Turn off color for downstairs dump Removing existing region "/var/tmp/test_repair" Creating region directory at: /var/tmp/test_repair Creating dsc directory at: /tmp/test_repair/test_repair_dsc Downstairs region 0 created at /var/tmp/test_repair/8810 in 0.19742145 Downstairs region 1 created at /var/tmp/test_repair/8820 in 0.01299213 Downstairs region 2 created at /var/tmp/test_repair/8830 in 0.012477259 Created 3 regions /var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1134 1135 1136 Dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:32.952063538Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:32.967800552Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:32.970444566Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:32.970510716Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:32.970519956Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:32.970531896Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139} {"msg":"Crucible 3f82f763-c7d1-4ed7-a5f3-0aa2f8098d8d has session id: 7625657d-f625-40d2-a483-70bfb0fefd91","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:32.971436818Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139,"session_id":"7625657d-f625-40d2-a483-70bfb0fefd91"} {"msg":"Upstairs opts: Upstairs UUID: 3f82f763-c7d1-4ed7-a5f3-0aa2f8098d8d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:32.971735628Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139,"session_id":"7625657d-f625-40d2-a483-70bfb0fefd91"} {"msg":"Crucible stats registered with UUID: 3f82f763-c7d1-4ed7-a5f3-0aa2f8098d8d","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:32.971747068Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139,"session_id":"7625657d-f625-40d2-a483-70bfb0fefd91"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:19:32.971759848Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139,"session_id":"7625657d-f625-40d2-a483-70bfb0fefd91"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:32.972389469Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139,"":"io task","client":"2","":"downstairs","session_id":"7625657d-f625-40d2-a483-70bfb0fefd91"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:32.972649189Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139,"":"io task","client":"1","":"downstairs","session_id":"7625657d-f625-40d2-a483-70bfb0fefd91"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:32.972664279Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139,"":"io task","client":"0","":"downstairs","session_id":"7625657d-f625-40d2-a483-70bfb0fefd91"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:32.97294068Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139,"task":"crutest"} {"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:32.97299531Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139} {"msg":"3f82f763-c7d1-4ed7-a5f3-0aa2f8098d8d active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:32.97300417Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139,"session_id":"7625657d-f625-40d2-a483-70bfb0fefd91"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:32.97301322Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139,"client":"0","":"downstairs","session_id":"7625657d-f625-40d2-a483-70bfb0fefd91"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:32.97302718Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139,"client":"1","":"downstairs","session_id":"7625657d-f625-40d2-a483-70bfb0fefd91"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:32.97303709Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139,"client":"2","":"downstairs","session_id":"7625657d-f625-40d2-a483-70bfb0fefd91"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:32.9730468Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139,"":"io task","client":"2","":"downstairs","session_id":"7625657d-f625-40d2-a483-70bfb0fefd91"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:32.97305741Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139,"":"io task","client":"0","":"downstairs","session_id":"7625657d-f625-40d2-a483-70bfb0fefd91"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:32.97306758Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139,"":"io task","client":"1","":"downstairs","session_id":"7625657d-f625-40d2-a483-70bfb0fefd91"} {"msg":"ds_connection connected from Ok(127.0.0.1:39781)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:32.97315581Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139,"":"io task","client":"2","":"downstairs","session_id":"7625657d-f625-40d2-a483-70bfb0fefd91"} {"msg":"ds_connection connected from Ok(127.0.0.1:60377)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:32.97316691Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139,"":"io task","client":"0","":"downstairs","session_id":"7625657d-f625-40d2-a483-70bfb0fefd91"} {"msg":"ds_connection connected from Ok(127.0.0.1:53430)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:32.97317983Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139,"":"io task","client":"1","":"downstairs","session_id":"7625657d-f625-40d2-a483-70bfb0fefd91"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:32.975340093Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139,"client":"1","":"downstairs","session_id":"7625657d-f625-40d2-a483-70bfb0fefd91"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:32.975362914Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139,"client":"0","":"downstairs","session_id":"7625657d-f625-40d2-a483-70bfb0fefd91"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:32.975376184Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139,"client":"2","":"downstairs","session_id":"7625657d-f625-40d2-a483-70bfb0fefd91"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:32.976183475Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139,"session_id":"7625657d-f625-40d2-a483-70bfb0fefd91"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:32.976206325Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139,"session_id":"7625657d-f625-40d2-a483-70bfb0fefd91"} {"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:32.976288875Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139,"":"downstairs","":"downstairs","session_id":"7625657d-f625-40d2-a483-70bfb0fefd91"} {"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:32.976300325Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139,"":"downstairs","":"downstairs","session_id":"7625657d-f625-40d2-a483-70bfb0fefd91"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:32.976311615Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139,"":"downstairs","":"downstairs","session_id":"7625657d-f625-40d2-a483-70bfb0fefd91"} {"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:32.976322005Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139,"":"downstairs","":"downstairs","session_id":"7625657d-f625-40d2-a483-70bfb0fefd91"} {"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:32.976332135Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139,"":"downstairs","":"downstairs","session_id":"7625657d-f625-40d2-a483-70bfb0fefd91"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:32.976343085Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139,"":"downstairs","":"downstairs","session_id":"7625657d-f625-40d2-a483-70bfb0fefd91"} {"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:32.976353465Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139,"":"downstairs","":"downstairs","session_id":"7625657d-f625-40d2-a483-70bfb0fefd91"} {"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:32.976363725Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139,"":"downstairs","":"downstairs","session_id":"7625657d-f625-40d2-a483-70bfb0fefd91"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:32.976374865Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139,"":"downstairs","":"downstairs","session_id":"7625657d-f625-40d2-a483-70bfb0fefd91"} {"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:32.976384255Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139,"":"downstairs","":"downstairs","session_id":"7625657d-f625-40d2-a483-70bfb0fefd91"} {"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:32.976394035Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139,"":"downstairs","":"downstairs","session_id":"7625657d-f625-40d2-a483-70bfb0fefd91"} {"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:32.976403495Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139,"":"downstairs","":"downstairs","session_id":"7625657d-f625-40d2-a483-70bfb0fefd91"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:32.976415755Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139,"":"downstairs","":"downstairs","session_id":"7625657d-f625-40d2-a483-70bfb0fefd91"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:32.976425575Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139,"session_id":"7625657d-f625-40d2-a483-70bfb0fefd91"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:32.976434415Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139,"client":"0","":"downstairs","session_id":"7625657d-f625-40d2-a483-70bfb0fefd91"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:32.976444345Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139,"client":"1","":"downstairs","session_id":"7625657d-f625-40d2-a483-70bfb0fefd91"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:32.976454085Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139,"client":"2","":"downstairs","session_id":"7625657d-f625-40d2-a483-70bfb0fefd91"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:32.976463915Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139,"session_id":"7625657d-f625-40d2-a483-70bfb0fefd91"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:32.976473135Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139,"session_id":"7625657d-f625-40d2-a483-70bfb0fefd91"} {"msg":"3f82f763-c7d1-4ed7-a5f3-0aa2f8098d8d is now active with session: 7625657d-f625-40d2-a483-70bfb0fefd91","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:32.976483185Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139,"session_id":"7625657d-f625-40d2-a483-70bfb0fefd91"} {"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:32.976492585Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:32.976500275Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139,"session_id":"7625657d-f625-40d2-a483-70bfb0fefd91"} {"msg":"Activated sub_volume 3f82f763-c7d1-4ed7-a5f3-0aa2f8098d8d","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:32.976898716Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:32.976915276Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Fill test","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:32.977303486Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139,"task":"crutest"} Read and Verify all blocks (0..600 range:false) {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:3 States: Active Active Active ,"JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY time":"2026-07-22T22:19:33.026678251Z","hostname":" 1004 Acked Read 256w-01KY5YGK9V792E6WY6KV93YHRV" Done Done Done," falsepid ": 1005 Acked Read 88 Done Done1139 Done false ," 1006 Acked Read 256task" Done: Done" Done falsecrutest" } STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 { 0 " Done msg 3 ": 3 " 3 9 Skipped 0 0 0 CLIENT: Tests done. All submitted work has been ACK'd 0 "," Error v": 0 0 0 ," 0 0 name":"Last Flush: crucible","Some(JobId(1003)) levelSome(JobId(1003)) "Some(JobId(1003)) : 30Downstairs last five completed: 1003 1002 1001 1000 ,"Upstairs last five completed: time" 1006: 1005" 1004 1003 1002 2026-07-22T22:19:33.026744981Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139,"task":"crutest"} {"msg":"CLIENT: Up:0 ds:3 act:3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:33.026777631Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139,"task":"crutest"} ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1007 1006 1005 1004 1003 {"msg":"CLIENT: Up:0 ds:0 act:3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.027963229Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139,"task":"crutest"} {"msg":"CLIENT: All crucible jobs finished, exiting program","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.028005979Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139,"task":"crutest"} {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:19:37.028016459Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139} {"msg":"client stopping due to Err(RecvError(()))","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.028064669Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139,"":"io task","client":"0","":"downstairs","session_id":"7625657d-f625-40d2-a483-70bfb0fefd91"} {"msg":"client stopping due to Err(RecvError(()))","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.028078609Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139,"":"io task","client":"1","":"downstairs","session_id":"7625657d-f625-40d2-a483-70bfb0fefd91"} {"msg":"client task is sending Done(QueueClosed)","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:19:37.028089919Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139,"":"io task","client":"1","":"downstairs","session_id":"7625657d-f625-40d2-a483-70bfb0fefd91"} {"msg":"client task could not reply to main task; shutting down?","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:19:37.028101119Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139,"":"io task","client":"1","":"downstairs","session_id":"7625657d-f625-40d2-a483-70bfb0fefd91"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.028111499Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139,"":"io task","client":"1","":"downstairs","session_id":"7625657d-f625-40d2-a483-70bfb0fefd91"} {"msg":"client task is sending Done(QueueClosed)","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:19:37.028161739Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139,"":"io task","client":"0","":"downstairs","session_id":"7625657d-f625-40d2-a483-70bfb0fefd91"} {"msg":"client task could not reply to main task; shutting down?","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:19:37.028172859Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139,"":"io task","client":"0","":"downstairs","session_id":"7625657d-f625-40d2-a483-70bfb0fefd91"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.028182779Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1139,"":"io task","client":"0","":"downstairs","session_id":"7625657d-f625-40d2-a483-70bfb0fefd91"} Begin loop 1 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.052968397Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1141,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:19:37.060175208Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1141,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.060230628Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1141,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.067571809Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1141,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.067869849Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1141} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.067898129Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1141} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.067906549Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1141} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.067918439Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1141} {"msg":"Crucible 3123af43-88f1-4a2d-b32b-cbad5b48d6c8 has session id: c03fad0f-3183-48f8-af0f-de761f98056c","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.06828825Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1141,"session_id":"c03fad0f-3183-48f8-af0f-de761f98056c"} {"msg":"Upstairs opts: Upstairs UUID: 3123af43-88f1-4a2d-b32b-cbad5b48d6c8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.06831722Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1141,"session_id":"c03fad0f-3183-48f8-af0f-de761f98056c"} {"msg":"Crucible stats registered with UUID: 3123af43-88f1-4a2d-b32b-cbad5b48d6c8","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.06832825Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1141,"session_id":"c03fad0f-3183-48f8-af0f-de761f98056c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:19:37.06833753Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1141,"session_id":"c03fad0f-3183-48f8-af0f-de761f98056c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.06847187Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1141,"":"io task","client":"2","":"downstairs","session_id":"c03fad0f-3183-48f8-af0f-de761f98056c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.06849029Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1141,"":"io task","client":"0","":"downstairs","session_id":"c03fad0f-3183-48f8-af0f-de761f98056c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.06850056Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1141,"":"io task","client":"1","":"downstairs","session_id":"c03fad0f-3183-48f8-af0f-de761f98056c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.06866677Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1141,"task":"crutest"} {"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.06868371Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1141} {"msg":"3123af43-88f1-4a2d-b32b-cbad5b48d6c8 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.06870275Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1141,"session_id":"c03fad0f-3183-48f8-af0f-de761f98056c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.06871187Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1141,"client":"0","":"downstairs","session_id":"c03fad0f-3183-48f8-af0f-de761f98056c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.06872157Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1141,"client":"1","":"downstairs","session_id":"c03fad0f-3183-48f8-af0f-de761f98056c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.06873127Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1141,"client":"2","":"downstairs","session_id":"c03fad0f-3183-48f8-af0f-de761f98056c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.06874081Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1141,"":"io task","client":"2","":"downstairs","session_id":"c03fad0f-3183-48f8-af0f-de761f98056c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.06875092Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1141,"":"io task","client":"0","":"downstairs","session_id":"c03fad0f-3183-48f8-af0f-de761f98056c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.068780931Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1141,"":"io task","client":"1","":"downstairs","session_id":"c03fad0f-3183-48f8-af0f-de761f98056c"} {"msg":"ds_connection connected from Ok(127.0.0.1:44456)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.068816181Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1141,"":"io task","client":"1","":"downstairs","session_id":"c03fad0f-3183-48f8-af0f-de761f98056c"} {"msg":"ds_connection connected from Ok(127.0.0.1:59891)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.068827531Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1141,"":"io task","client":"2","":"downstairs","session_id":"c03fad0f-3183-48f8-af0f-de761f98056c"} {"msg":"ds_connection connected from Ok(127.0.0.1:60887)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.068890171Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1141,"":"io task","client":"0","":"downstairs","session_id":"c03fad0f-3183-48f8-af0f-de761f98056c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.069561022Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1141,"client":"1","":"downstairs","session_id":"c03fad0f-3183-48f8-af0f-de761f98056c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.069587842Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1141,"client":"0","":"downstairs","session_id":"c03fad0f-3183-48f8-af0f-de761f98056c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.069629622Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1141,"client":"2","":"downstairs","session_id":"c03fad0f-3183-48f8-af0f-de761f98056c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.069877922Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1141,"session_id":"c03fad0f-3183-48f8-af0f-de761f98056c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.069902392Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1141,"session_id":"c03fad0f-3183-48f8-af0f-de761f98056c"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.069955222Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1141,"":"downstairs","":"downstairs","session_id":"c03fad0f-3183-48f8-af0f-de761f98056c"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.069966822Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1141,"":"downstairs","":"downstairs","session_id":"c03fad0f-3183-48f8-af0f-de761f98056c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.069978162Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1141,"":"downstairs","":"downstairs","session_id":"c03fad0f-3183-48f8-af0f-de761f98056c"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.069988582Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1141,"":"downstairs","":"downstairs","session_id":"c03fad0f-3183-48f8-af0f-de761f98056c"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.070011722Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1141,"":"downstairs","":"downstairs","session_id":"c03fad0f-3183-48f8-af0f-de761f98056c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.070022792Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1141,"":"downstairs","":"downstairs","session_id":"c03fad0f-3183-48f8-af0f-de761f98056c"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.070033132Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1141,"":"downstairs","":"downstairs","session_id":"c03fad0f-3183-48f8-af0f-de761f98056c"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.070043522Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1141,"":"downstairs","":"downstairs","session_id":"c03fad0f-3183-48f8-af0f-de761f98056c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.070054562Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1141,"":"downstairs","":"downstairs","session_id":"c03fad0f-3183-48f8-af0f-de761f98056c"} {"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.070064112Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1141,"":"downstairs","":"downstairs","session_id":"c03fad0f-3183-48f8-af0f-de761f98056c"} {"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.070073962Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1141,"":"downstairs","":"downstairs","session_id":"c03fad0f-3183-48f8-af0f-de761f98056c"} {"msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.070083423Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1141,"":"downstairs","":"downstairs","session_id":"c03fad0f-3183-48f8-af0f-de761f98056c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.070092783Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1141,"":"downstairs","":"downstairs","session_id":"c03fad0f-3183-48f8-af0f-de761f98056c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.070102583Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1141,"session_id":"c03fad0f-3183-48f8-af0f-de761f98056c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.070111443Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1141,"client":"0","":"downstairs","session_id":"c03fad0f-3183-48f8-af0f-de761f98056c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.070122833Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1141,"client":"1","":"downstairs","session_id":"c03fad0f-3183-48f8-af0f-de761f98056c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.070132583Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1141,"client":"2","":"downstairs","session_id":"c03fad0f-3183-48f8-af0f-de761f98056c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.070142643Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1141,"session_id":"c03fad0f-3183-48f8-af0f-de761f98056c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.070151843Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1141,"session_id":"c03fad0f-3183-48f8-af0f-de761f98056c"} {"msg":"3123af43-88f1-4a2d-b32b-cbad5b48d6c8 is now active with session: c03fad0f-3183-48f8-af0f-de761f98056c","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.070161833Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1141,"session_id":"c03fad0f-3183-48f8-af0f-de761f98056c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.070170743Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1141,"session_id":"c03fad0f-3183-48f8-af0f-de761f98056c"} {"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.070180033Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1141} {"msg":"Activated sub_volume 3123af43-88f1-4a2d-b32b-cbad5b48d6c8","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.070188163Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1141} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.070196323Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1141,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.070719023Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1141,"task":"crutest"} 01/30 Write block 358 len 16384 data: 2 2 2 2 02/30 Flush 03/30 Write block 144 len 4096 data: 2 04/30 Read block 570 len 24576 05/30 Read block 274 len 20480 06/30 Write block 280 len 4096 data: 2 07/30 Read block 201 len 28672 08/30 Write block 492 len 12288 data: 2 2 2 09/30 Write block 489 len 16384 data: 2 2 2 3 10/30 Read block 431 len 24576 11/30 Flush 12/30 Write block 216 len 8192 data: 2 2 13/30 Read block 184 len 16384 14/30 Write block 30 len 28672 data: 2 2 2 2 2 2 2 15/30 Read block 474 len 36864 16/30 Write block 359 len 16384 data: 3 3 3 2 17/30 Read block 44 len 36864 18/30 Read block 431 len 32768 19/30 Write block 110 len 16384 data: 2 2 2 2 20/30 Read block 7 len 32768 21/30 Write block 448 len 36864 data: 2 2 2 2 2 2 2 2 2 22/30 Write block 145 len 36864 data: 2 2 2 2 2 2 2 2 2 23/30 Write block 0 len 36864 data: 2 2 2 2 2 2 2 2 2 24/30 Flush 25/30 Write block 106 len 36864 data: 2 2 2 2 3 3 3 3 2 26/30 Read block 231 len 28672 27/30 Write block 466 len 16384 data: 2 2 2 2 28/30 Write block 149 len 4096 data: 3 29/30 Write block 45 len 36864 data: 2 2 2 2 2 2 2 2 2 30/30 Write block 59 len 28672 data: 2 2 2 2 2 2 2 ---------------------------------------------------------------- Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 1 Done Done Done false 1003 Acked Read 6 Done Done Done false 1004 Acked Read 5 Sent Done Done false 1005 Acked Write 1 Sent Done Done false 1006 Acked Read 7 Sent Done Done false 1007 Acked Write 3 Sent Done Done false 1008 Acked Write 4 Sent Done Done false 1009 Acked Read 6 Sent Done Done false 1010 Acked Flush 0 Sent Done Done false 1011 Acked Write 2 Sent Done Done false 1012 Acked Read 4 Sent Done Done false 1013 Acked Write 7 Sent Done Done false 1014 Acked Read 9 Sent Done Done false 1015 Acked Write 4 Sent Done Done false 1016 Acked Read 9 Sent Done Done false 1017 Acked Read 8 Sent Done Done false 1018 Acked Write 4 Sent Done Done false 1019 Acked Read 8 Sent Done Done false 1020 Acked Write 9 Sent Done Done false 1021 Acked Write 9 Sent Done Done false 1022 Acked Write 9 Sent Done Done false 1023 Acked Flush 0 Sent Done Done false 1024 Acked Write 9 Sent Done Done false 1025 Acked Read 7 Sent Done Done false 1026 Acked Write 4 Sent Done Done false 1027 Acked Write 1 Sent Done Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 26 2 2 30 Done 2 26 26 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1001)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:19:37.079459867Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1141} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:37.079655347Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1141,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 2 2 1 4 4 F F F <--- 1 020-039 1 2 2 1 4 4 F F F <--- 2 040-059 1 1 1 1 1 1 F T T <--- 3 060-079 1 1 1 1 1 1 F T T <--- 4 080-099 1 1 1 1 1 1 F F F 5 100-119 1 2 2 1 4 4 F T T <--- 6 120-139 1 1 1 1 1 1 F F F 7 140-159 1 2 2 1 4 4 T T T <--- 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 2 2 1 4 4 F F F <--- 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 1 2 2 1 3 3 F F F <--- 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 2 2 2 2 4 4 F F F <--- 18 360-379 2 2 2 2 4 4 F F F <--- 19 380-399 1 1 1 1 1 1 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 1 2 2 1 4 4 F F F <--- 23 460-479 1 1 1 1 1 1 F T T <--- 24 480-499 1 2 2 1 3 3 F F F <--- 25 500-519 1 1 1 1 1 1 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 2, Max flush: 4 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 1 Verifying data now No change after verify Loop: 1 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 4 4 4 F F F 1 020-039 2 2 2 4 4 4 F F F 2 040-059 3 3 3 5 5 5 F F F 3 060-079 3 3 3 5 5 5 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 3 3 3 5 5 5 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 3 3 3 5 5 5 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 2 2 2 4 4 4 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 2 2 2 3 3 3 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 2 2 2 4 4 4 F F F 18 360-379 2 2 2 4 4 4 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 2 2 2 4 4 4 F F F 23 460-479 3 3 3 5 5 5 F F F 24 480-499 2 2 2 3 3 3 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 3, Max flush: 5 Begin loop 2 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:41.464143153Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1149,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:19:41.471313744Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1149,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:41.471404344Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1149,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:41.478145154Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1149,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:41.478412315Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1149} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:41.478442505Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1149} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:41.478451345Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1149} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:41.478466505Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1149} {"msg":"Crucible a3993eeb-0cfb-4352-8c6a-5c6094ecb6b3 has session id: 9695b565-ed85-44e8-b49a-574f926eef75","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:41.478788345Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1149,"session_id":"9695b565-ed85-44e8-b49a-574f926eef75"} {"msg":"Upstairs opts: Upstairs UUID: a3993eeb-0cfb-4352-8c6a-5c6094ecb6b3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:41.478817895Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1149,"session_id":"9695b565-ed85-44e8-b49a-574f926eef75"} {"msg":"Crucible stats registered with UUID: a3993eeb-0cfb-4352-8c6a-5c6094ecb6b3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:41.478829266Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1149,"session_id":"9695b565-ed85-44e8-b49a-574f926eef75"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:19:41.478838666Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1149,"session_id":"9695b565-ed85-44e8-b49a-574f926eef75"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:41.478979336Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1149,"":"io task","client":"1","":"downstairs","session_id":"9695b565-ed85-44e8-b49a-574f926eef75"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:41.478997126Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1149,"":"io task","client":"2","":"downstairs","session_id":"9695b565-ed85-44e8-b49a-574f926eef75"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:41.479007416Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1149,"":"io task","client":"0","":"downstairs","session_id":"9695b565-ed85-44e8-b49a-574f926eef75"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:41.479143496Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1149,"task":"crutest"} {"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:41.479167546Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1149} {"msg":"a3993eeb-0cfb-4352-8c6a-5c6094ecb6b3 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:41.479180386Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1149,"session_id":"9695b565-ed85-44e8-b49a-574f926eef75"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:41.479189526Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1149,"client":"0","":"downstairs","session_id":"9695b565-ed85-44e8-b49a-574f926eef75"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:41.479199206Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1149,"client":"1","":"downstairs","session_id":"9695b565-ed85-44e8-b49a-574f926eef75"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:41.479208846Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1149,"client":"2","":"downstairs","session_id":"9695b565-ed85-44e8-b49a-574f926eef75"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:41.479218566Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1149,"":"io task","client":"2","":"downstairs","session_id":"9695b565-ed85-44e8-b49a-574f926eef75"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:41.479230536Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1149,"":"io task","client":"0","":"downstairs","session_id":"9695b565-ed85-44e8-b49a-574f926eef75"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:41.479240546Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1149,"":"io task","client":"1","":"downstairs","session_id":"9695b565-ed85-44e8-b49a-574f926eef75"} {"msg":"ds_connection connected from Ok(127.0.0.1:57966)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:41.479347376Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1149,"":"io task","client":"1","":"downstairs","session_id":"9695b565-ed85-44e8-b49a-574f926eef75"} {"msg":"ds_connection connected from Ok(127.0.0.1:53208)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:41.479406536Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1149,"":"io task","client":"2","":"downstairs","session_id":"9695b565-ed85-44e8-b49a-574f926eef75"} {"msg":"ds_connection connected from Ok(127.0.0.1:47740)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:41.479418506Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1149,"":"io task","client":"0","":"downstairs","session_id":"9695b565-ed85-44e8-b49a-574f926eef75"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:41.479946397Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1149,"client":"2","":"downstairs","session_id":"9695b565-ed85-44e8-b49a-574f926eef75"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:41.479965447Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1149,"client":"1","":"downstairs","session_id":"9695b565-ed85-44e8-b49a-574f926eef75"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:41.480079407Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1149,"client":"0","":"downstairs","session_id":"9695b565-ed85-44e8-b49a-574f926eef75"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:41.480270108Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1149,"session_id":"9695b565-ed85-44e8-b49a-574f926eef75"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:41.480289298Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1149,"session_id":"9695b565-ed85-44e8-b49a-574f926eef75"} {"msg":"[0]R flush_numbers[0..12]: [4, 4, 5, 5, 1, 5, 1, 5, 1, 1, 4, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:41.480344418Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1149,"":"downstairs","":"downstairs","session_id":"9695b565-ed85-44e8-b49a-574f926eef75"} {"msg":"[0]R generation[0..12]: [2, 2, 3, 3, 1, 3, 1, 3, 1, 1, 2, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:41.480355438Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1149,"":"downstairs","":"downstairs","session_id":"9695b565-ed85-44e8-b49a-574f926eef75"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:41.480366638Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1149,"":"downstairs","":"downstairs","session_id":"9695b565-ed85-44e8-b49a-574f926eef75"} {"msg":"[1]R flush_numbers[0..12]: [4, 4, 5, 5, 1, 5, 1, 5, 1, 1, 4, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:41.480376938Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1149,"":"downstairs","":"downstairs","session_id":"9695b565-ed85-44e8-b49a-574f926eef75"} {"msg":"[1]R generation[0..12]: [2, 2, 3, 3, 1, 3, 1, 3, 1, 1, 2, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:41.480387178Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1149,"":"downstairs","":"downstairs","session_id":"9695b565-ed85-44e8-b49a-574f926eef75"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:41.480400428Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1149,"":"downstairs","":"downstairs","session_id":"9695b565-ed85-44e8-b49a-574f926eef75"} {"msg":"[2]R flush_numbers[0..12]: [4, 4, 5, 5, 1, 5, 1, 5, 1, 1, 4, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:41.480410878Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1149,"":"downstairs","":"downstairs","session_id":"9695b565-ed85-44e8-b49a-574f926eef75"} {"msg":"[2]R generation[0..12]: [2, 2, 3, 3, 1, 3, 1, 3, 1, 1, 2, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:41.480421098Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1149,"":"downstairs","":"downstairs","session_id":"9695b565-ed85-44e8-b49a-574f926eef75"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:41.480432218Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1149,"":"downstairs","":"downstairs","session_id":"9695b565-ed85-44e8-b49a-574f926eef75"} {"msg":"Max found gen is 4","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:41.480441868Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1149,"":"downstairs","":"downstairs","session_id":"9695b565-ed85-44e8-b49a-574f926eef75"} {"msg":"Generation requested: 4 >= found:4","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:41.480451708Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1149,"":"downstairs","":"downstairs","session_id":"9695b565-ed85-44e8-b49a-574f926eef75"} {"msg":"Next flush: 6","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:41.480461098Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1149,"":"downstairs","":"downstairs","session_id":"9695b565-ed85-44e8-b49a-574f926eef75"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:41.480470688Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1149,"":"downstairs","":"downstairs","session_id":"9695b565-ed85-44e8-b49a-574f926eef75"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:41.480480918Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1149,"session_id":"9695b565-ed85-44e8-b49a-574f926eef75"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:41.480489878Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1149,"client":"0","":"downstairs","session_id":"9695b565-ed85-44e8-b49a-574f926eef75"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:41.480499898Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1149,"client":"1","":"downstairs","session_id":"9695b565-ed85-44e8-b49a-574f926eef75"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:41.480510018Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1149,"client":"2","":"downstairs","session_id":"9695b565-ed85-44e8-b49a-574f926eef75"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:41.480520218Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1149,"session_id":"9695b565-ed85-44e8-b49a-574f926eef75"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:41.480529668Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1149,"session_id":"9695b565-ed85-44e8-b49a-574f926eef75"} {"msg":"a3993eeb-0cfb-4352-8c6a-5c6094ecb6b3 is now active with session: 9695b565-ed85-44e8-b49a-574f926eef75","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:41.480539578Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1149,"session_id":"9695b565-ed85-44e8-b49a-574f926eef75"} {"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:41.480549988Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1149} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:41.480557678Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1149,"session_id":"9695b565-ed85-44e8-b49a-574f926eef75"} {"msg":"Activated sub_volume a3993eeb-0cfb-4352-8c6a-5c6094ecb6b3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:41.480567008Z","hostname":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 w-01KY5YGK9V792E6WY6KV93YHRV","pid":1149} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:41.480580158Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1149,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:41.480814059Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1149,"task":"crutest"} 01/30 Write block 293 len 32768 data: 2 2 2 2 2 2 2 2 02/30 Write block 312 len 8192 data: 2 2 03/30 Write block 417 len 32768 data: 2 2 2 2 2 2 2 2 04/30 Read block 189 len 32768 05/30 Read block 78 len 28672 06/30 Read block 125 len 12288 07/30 Read block 537 len 32768 08/30 Write block 212 len 8192 data: 2 2 09/30 Flush 10/30 Write block 322 len 8192 data: 2 2 11/30 Read block 557 len 8192 12/30 Read block 180 len 36864 13/30 Read block 22 len 16384 14/30 Write block 269 len 36864 data: 2 2 2 2 2 2 2 2 2 15/30 Write block 323 len 36864 data: 3 2 2 2 2 2 2 2 2 16/30 Read block 179 len 36864 17/30 Read block 418 len 4096 18/30 Read block 239 len 12288 19/30 Write block 301 len 12288 data: 2 2 2 20/30 Read block 99 len 32768 21/30 Write block 300 len 32768 data: 3 3 3 3 2 2 2 2 22/30 Write block 39 len 20480 data: 2 2 2 2 2 23/30 Read block 270 len 8192 24/30 Read block 325 len 32768 25/30 Read block 405 len 16384 26/30 Read block 123 len 20480 27/30 Read block 445 len 24576 28/30 Read block 376 len 12288 29/30 Read block 78 len 40960 30/30 Read block 409 len 36864 ---------------------------------------------------------------- Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Write 2 Done Done Done false 1002 Acked Write 8 Done Done Done false 1003 Acked Read 8 Done Done Done false 1004 Acked Read 7 Done Done Done false 1005 Acked Read 3 Sent Done Done false 1006 Acked Read 8 Sent Done Done false 1007 Acked Write 2 Sent Done Done false 1008 Acked Flush 0 Sent Done Done false 1009 Acked Write 2 Sent Done Done false 1010 Acked Read 2 Sent Done Done false 1011 Acked Read 9 Sent Done Done false 1012 Acked Read 4 Sent Done Done false 1013 Acked Write 9 Sent Done Done false 1014 Acked Write 9 Sent Done Done false 1015 Acked Read 9 Sent Done Done false 1016 Acked Read 1 Sent Done Done false 1017 Acked Read 3 Sent Done Done false 1018 Acked Write 3 Sent Done Done false 1019 Acked Read 8 Sent Done Done false 1020 Acked Write 8 Sent Done Done false 1021 Acked Write 5 Sent Done Done false 1022 Acked Read 2 Sent Done Done false 1023 Acked Read 8 Sent Done Done false 1024 Acked Read 4 Sent Done Done false 1025 Acked Read 5 Sent Done Done false 1026 Acked Read 6 Sent Done Done false 1027 Acked Read 3 Sent Done Done false 1028 Acked Read 10 Sent Done Done false 1029 Acked Read 9 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 25 1 0 26 Done 5 29 30 64 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1008)) Some(JobId(1008)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:19:41.489694802Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1149} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:41.489870132Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1149,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 4 4 4 F F F 1 020-039 2 2 2 4 4 4 F T T <--- 2 040-059 3 3 3 5 5 5 F T T <--- 3 060-079 3 3 3 5 5 5 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 3 3 3 5 5 5 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 3 3 3 5 5 5 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 2 4 4 4 6 6 F F F <--- 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 1 1 1 1 1 1 F T T <--- 14 280-299 2 4 4 3 6 6 T F F <--- 15 300-319 1 4 4 1 6 6 T T T <--- 16 320-339 1 1 1 1 1 1 F T T <--- 17 340-359 2 2 2 4 4 4 F F F 18 360-379 2 2 2 4 4 4 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 1 4 4 1 6 6 T F F <--- 21 420-439 1 4 4 1 6 6 T F F <--- 22 440-459 2 2 2 4 4 4 F F F 23 460-479 3 3 3 5 5 5 F F F 24 480-499 2 2 2 3 3 3 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 4, Max flush: 6 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 2 Verifying data now No change after verify Loop: 2 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 4 4 4 F F F 1 020-039 5 5 5 7 7 7 F F F 2 040-059 5 5 5 7 7 7 F F F 3 060-079 3 3 3 5 5 5 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 3 3 3 5 5 5 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 3 3 3 5 5 5 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 4 4 4 6 6 6 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 5 5 5 7 7 7 F F F 14 280-299 4 4 4 6 6 6 F F F 15 300-319 5 5 5 7 7 7 F F F 16 320-339 5 5 5 7 7 7 F F F 17 340-359 2 2 2 4 4 4 F F F 18 360-379 2 2 2 4 4 4 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 4 4 4 6 6 6 F F F 21 420-439 4 4 4 6 6 6 F F F 22 440-459 2 2 2 4 4 4 F F F 23 460-479 3 3 3 5 5 5 F F F 24 480-499 2 2 2 3 3 3 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 5, Max flush: 7 Begin loop 3 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:45.729156189Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1157,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:45.737591182Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1157,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:45.737888223Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1157} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:45.737912993Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1157} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:45.737921613Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1157} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:45.737933373Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1157} {"msg":"Crucible fd2b09ed-e068-40ab-a6a7-88e53dfe0aa2 has session id: 016c148f-6baf-43d8-a91f-b99b6febb54b","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:45.738226003Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1157,"session_id":"016c148f-6baf-43d8-a91f-b99b6febb54b"} {"msg":"Upstairs opts: Upstairs UUID: fd2b09ed-e068-40ab-a6a7-88e53dfe0aa2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:45.738275983Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1157,"session_id":"016c148f-6baf-43d8-a91f-b99b6febb54b"} {"msg":"Crucible stats registered with UUID: fd2b09ed-e068-40ab-a6a7-88e53dfe0aa2","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:45.738287103Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1157,"session_id":"016c148f-6baf-43d8-a91f-b99b6febb54b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:19:45.738296443Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1157,"session_id":"016c148f-6baf-43d8-a91f-b99b6febb54b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:45.738423233Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1157,"":"io task","client":"1","":"downstairs","session_id":"016c148f-6baf-43d8-a91f-b99b6febb54b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:45.738440713Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1157,"":"io task","client":"0","":"downstairs","session_id":"016c148f-6baf-43d8-a91f-b99b6febb54b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:45.738450783Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1157,"":"io task","client":"2","":"downstairs","session_id":"016c148f-6baf-43d8-a91f-b99b6febb54b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:45.738606384Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1157,"task":"crutest"} {"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:45.738622924Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1157} {"msg":"fd2b09ed-e068-40ab-a6a7-88e53dfe0aa2 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:45.738656744Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1157,"session_id":"016c148f-6baf-43d8-a91f-b99b6febb54b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:45.738666184Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1157,"client":"0","":"downstairs","session_id":"016c148f-6baf-43d8-a91f-b99b6febb54b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:45.738676044Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1157,"client":"1","":"downstairs","session_id":"016c148f-6baf-43d8-a91f-b99b6febb54b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:45.738685964Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1157,"client":"2","":"downstairs","session_id":"016c148f-6baf-43d8-a91f-b99b6febb54b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:45.738695694Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1157,"":"io task","client":"2","":"downstairs","session_id":"016c148f-6baf-43d8-a91f-b99b6febb54b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:45.738705764Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1157,"":"io task","client":"0","":"downstairs","session_id":"016c148f-6baf-43d8-a91f-b99b6febb54b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:45.738715784Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1157,"":"io task","client":"1","":"downstairs","session_id":"016c148f-6baf-43d8-a91f-b99b6febb54b"} {"msg":"ds_connection connected from Ok(127.0.0.1:40120)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:45.738820374Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1157,"":"io task","client":"1","":"downstairs","session_id":"016c148f-6baf-43d8-a91f-b99b6febb54b"} {"msg":"ds_connection connected from Ok(127.0.0.1:47114)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:45.738942264Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1157,"":"io task","client":"2","":"downstairs","session_id":"016c148f-6baf-43d8-a91f-b99b6febb54b"} {"msg":"ds_connection connected from Ok(127.0.0.1:41073)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:45.738956174Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1157,"":"io task","client":"0","":"downstairs","session_id":"016c148f-6baf-43d8-a91f-b99b6febb54b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:45.739464275Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1157,"client":"2","":"downstairs","session_id":"016c148f-6baf-43d8-a91f-b99b6febb54b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:45.739521435Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1157,"client":"1","":"downstairs","session_id":"016c148f-6baf-43d8-a91f-b99b6febb54b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:45.739573005Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1157,"client":"0","":"downstairs","session_id":"016c148f-6baf-43d8-a91f-b99b6febb54b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:45.739859896Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1157,"session_id":"016c148f-6baf-43d8-a91f-b99b6febb54b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:45.739870426Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1157,"session_id":"016c148f-6baf-43d8-a91f-b99b6febb54b"} {"msg":"[0]R flush_numbers[0..12]: [4, 7, 7, 5, 1, 5, 1, 5, 1, 1, 6, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:45.739903626Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1157,"":"downstairs","":"downstairs","session_id":"016c148f-6baf-43d8-a91f-b99b6febb54b"} {"msg":"[0]R generation[0..12]: [2, 5, 5, 3, 1, 3, 1, 3, 1, 1, 4, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:45.739914606Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1157,"":"downstairs","":"downstairs","session_id":"016c148f-6baf-43d8-a91f-b99b6febb54b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:45.739925816Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1157,"":"downstairs","":"downstairs","session_id":"016c148f-6baf-43d8-a91f-b99b6febb54b"} {"msg":"[1]R flush_numbers[0..12]: [4, 7, 7, 5, 1, 5, 1, 5, 1, 1, 6, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:45.739936046Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1157,"":"downstairs","":"downstairs","session_id":"016c148f-6baf-43d8-a91f-b99b6febb54b"} {"msg":"[1]R generation[0..12]: [2, 5, 5, 3, 1, 3, 1, 3, 1, 1, 4, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:45.739946376Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1157,"":"downstairs","":"downstairs","session_id":"016c148f-6baf-43d8-a91f-b99b6febb54b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:45.739957396Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1157,"":"downstairs","":"downstairs","session_id":"016c148f-6baf-43d8-a91f-b99b6febb54b"} {"msg":"[2]R flush_numbers[0..12]: [4, 7, 7, 5, 1, 5, 1, 5, 1, 1, 6, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:45.739967696Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1157,"":"downstairs","":"downstairs","session_id":"016c148f-6baf-43d8-a91f-b99b6febb54b"} {"msg":"[2]R generation[0..12]: [2, 5, 5, 3, 1, 3, 1, 3, 1, 1, 4, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:45.739977956Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1157,"":"downstairs","":"downstairs","session_id":"016c148f-6baf-43d8-a91f-b99b6febb54b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:45.739988946Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1157,"":"downstairs","":"downstairs","session_id":"016c148f-6baf-43d8-a91f-b99b6febb54b"} {"msg":"Max found gen is 6","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:45.739998356Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1157,"":"downstairs","":"downstairs","session_id":"016c148f-6baf-43d8-a91f-b99b6febb54b"} {"msg":"Generation requested: 6 >= found:6","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:45.740008156Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1157,"":"downstairs","":"downstairs","session_id":"016c148f-6baf-43d8-a91f-b99b6febb54b"} {"msg":"Next flush: 8","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:45.740019426Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1157,"":"downstairs","":"downstairs","session_id":"016c148f-6baf-43d8-a91f-b99b6febb54b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:45.740028966Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1157,"":"downstairs","":"downstairs","session_id":"016c148f-6baf-43d8-a91f-b99b6febb54b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:45.740038616Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1157,"session_id":"016c148f-6baf-43d8-a91f-b99b6febb54b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:45.740047556Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1157,"client":"0","":"downstairs","session_id":"016c148f-6baf-43d8-a91f-b99b6febb54b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:45.740057246Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1157,"client":"1","":"downstairs","session_id":"016c148f-6baf-43d8-a91f-b99b6febb54b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:45.740066916Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1157,"client":"2","":"downstairs","session_id":"016c148f-6baf-43d8-a91f-b99b6febb54b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:45.740076676Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1157,"session_id":"016c148f-6baf-43d8-a91f-b99b6febb54b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:45.740085916Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1157,"session_id":"016c148f-6baf-43d8-a91f-b99b6febb54b"} {"msg":"fd2b09ed-e068-40ab-a6a7-88e53dfe0aa2 is now active with session: 016c148f-6baf-43d8-a91f-b99b6febb54b","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:45.740095846Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1157,"session_id":"016c148f-6baf-43d8-a91f-b99b6febb54b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:45.740104596Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1157,"session_id":"016c148f-6baf-43d8-a91f-b99b6febb54b"} {"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:45.740113616Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1157} {"msg":"Activated sub_volume fd2b09ed-e068-40ab-a6a7-88e53dfe0aa2","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:45.740121696Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1157} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:45.740129876Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1157,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:45.740447476Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1157,"task":"crutest"} 01/30 Write block 268 len 40960 data: 2 3 3 3 3 3 3 3 3 3 02/30 Write block 266 len 16384 data: 2 2 3 4 03/30 Read block 574 len 36864 04/30 Write block 118 len 32768 data: 2 2 2 2 2 2 2 2 05/30 Write block 193 len 4096 data: 2 06/30 Read block 171 len 4096 07/30 Read block 364 len 12288 08/30 Read block 23 len 40960 09/30 Read block 589 len 16384 10/30 Write block 254 len 8192 data: 2 2 11/30 Write block 570 len 12288 data: 2 2 2 12/30 Write block 196 len 12288 data: 2 2 2 13/30 Read block 370 len 24576 14/30 Read block 352 len 20480 15/30 Write block 431 len 4096 data: 2 16/30 Read block 200 len 20480 17/30 Read block 536 len 36864 18/30 Write block 394 len 24576 data: 2 2 2 2 2 2 19/30 Read block 577 len 40960 20/30 Read block 264 len 4096 21/30 Read block 220 len 4096 22/30 Read block 93 len 28672 23/30 Write block 565 len 24576 data: 2 2 2 2 2 3 24/30 Read block 429 len 32768 25/30 Read block 377 len 32768 26/30 Read block 189 len 24576 27/30 Write block 577 len 8192 data: 2 2 28/30 Read block 579 len 32768 29/30 Read block 145 len 4096 30/30 Write block 388 len 20480 data: 2 2 2 2 2 ---------------------------------------------------------------- Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Write 4 Done Done Done false 1002 Acked Read 9 Done Sent Done false 1003 Acked Write 8 Done Sent Done false 1004 Acked Write 1 Done Sent Done false 1005 Acked Read 1 Done Sent Done false 1006 Acked Read 3 Done Sent Done false 1007 Acked Read 10 Done Sent Done false 1008 Acked Read 4 Done Sent Done false 1009 Acked Write 2 Done Sent Done false 1010 Acked Write 3 Done Sent Done false 1011 Acked Write 3 Done Sent Done false 1012 Acked Read 6 Done Sent Done false 1013 Acked Read 5 Done Sent Done false 1014 Acked Write 1 Done Sent Done false 1015 Acked Read 5 Done Sent Done false 1016 Acked Read 9 Done Sent Done false 1017 Acked Write 6 Done Sent Done false 1018 Acked Read 10 Done Sent Done false 1019 Acked Read 1 Done Sent Done false 1020 Acked Read 1 Done Sent Done false 1021 Acked Read 7 Done Sent Done false 1022 Acked Write 6 Done Sent Done false 1023 Acked Read 8 Done Sent Done false 1024 Acked Read 8 Done Sent Done false 1025 Acked Read 6 Done Sent Done false 1026 Acked Write 2 Done Sent Done false 1027 Acked Read 8 Done Sent Done false 1028 Acked Read 1 Done Sent Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 28 1 30 Done 29 2 29 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:19:45.748259538Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1157} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:45.748453128Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1157,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 4 4 4 F F F 1 020-039 5 5 5 7 7 7 F F F 2 040-059 5 5 5 7 7 7 F F F 3 060-079 3 3 3 5 5 5 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 3 3 3 5 5 5 T F T <--- 6 120-139 1 1 1 1 1 1 T F T <--- 7 140-159 3 3 3 5 5 5 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 T F T <--- 10 200-219 4 4 4 6 6 6 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 T F T <--- 13 260-279 5 5 5 7 7 7 T T T 14 280-299 4 4 4 6 6 6 F F F 15 300-319 5 5 5 7 7 7 F F F 16 320-339 5 5 5 7 7 7 F F F 17 340-359 2 2 2 4 4 4 F F F 18 360-379 2 2 2 4 4 4 F F F 19 380-399 1 1 1 1 1 1 T F T <--- 20 400-419 4 4 4 6 6 6 F F F 21 420-439 4 4 4 6 6 6 T F T <--- 22 440-459 2 2 2 4 4 4 F F F 23 460-479 3 3 3 5 5 5 F F F 24 480-499 2 2 2 3 3 3 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 1 1 1 1 1 1 T F T <--- 29 580-599 1 1 1 1 1 1 F F F Max gen: 5, Max flush: 7 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 3 Verifying data now No change after verify Loop: 3 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 4 4 4 F F F 1 020-039 5 5 5 7 7 7 F F F 2 040-059 5 5 5 7 7 7 F F F 3 060-079 3 3 3 5 5 5 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 6 6 6 8 8 8 F F F 6 120-139 6 6 6 8 8 8 F F F 7 140-159 3 3 3 5 5 5 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 6 6 6 8 8 8 F F F 10 200-219 4 4 4 6 6 6 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 6 6 6 8 8 8 F F F 13 260-279 6 6 6 8 8 8 F F F 14 280-299 4 4 4 6 6 6 F F F 15 300-319 5 5 5 7 7 7 F F F 16 320-339 5 5 5 7 7 7 F F F 17 340-359 2 2 2 4 4 4 F F F 18 360-379 2 2 2 4 4 4 F F F 19 380-399 6 6 6 8 8 8 F F F 20 400-419 4 4 4 6 6 6 F F F 21 420-439 6 6 6 8 8 8 F F F 22 440-459 2 2 2 4 4 4 F F F 23 460-479 3 3 3 5 5 5 F F F 24 480-499 2 2 2 3 3 3 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 6 6 6 8 8 8 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 6, Max flush: 8 Begin loop 4 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:49.982172467Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1165,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:49.990214399Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1165,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:49.990513359Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1165} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:49.990548749Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1165} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:49.990559859Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1165} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:49.990571779Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1165} {"msg":"Crucible c68a2df0-7cd9-413e-9c13-0addc2ecdfc6 has session id: 9110d401-07dc-4a95-af32-675b9320428e","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:49.99087854Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1165,"session_id":"9110d401-07dc-4a95-af32-675b9320428e"} {"msg":"Upstairs opts: Upstairs UUID: c68a2df0-7cd9-413e-9c13-0addc2ecdfc6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:49.99090074Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1165,"session_id":"9110d401-07dc-4a95-af32-675b9320428e"} {"msg":"Crucible stats registered with UUID: c68a2df0-7cd9-413e-9c13-0addc2ecdfc6","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:49.99091152Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1165,"session_id":"9110d401-07dc-4a95-af32-675b9320428e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:19:49.99092073Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1165,"session_id":"9110d401-07dc-4a95-af32-675b9320428e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:49.99104708Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1165,"":"io task","client":"0","":"downstairs","session_id":"9110d401-07dc-4a95-af32-675b9320428e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:49.99107902Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1165,"":"io task","client":"1","":"downstairs","session_id":"9110d401-07dc-4a95-af32-675b9320428e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:49.99108992Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1165,"":"io task","client":"2","":"downstairs","session_id":"9110d401-07dc-4a95-af32-675b9320428e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:49.99121107Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1165,"task":"crutest"} {"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:49.99122962Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1165} {"msg":"c68a2df0-7cd9-413e-9c13-0addc2ecdfc6 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:49.991245521Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1165,"session_id":"9110d401-07dc-4a95-af32-675b9320428e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:49.991254661Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1165,"client":"0","":"downstairs","session_id":"9110d401-07dc-4a95-af32-675b9320428e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:49.991264611Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1165,"client":"1","":"downstairs","session_id":"9110d401-07dc-4a95-af32-675b9320428e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:49.991276031Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1165,"client":"2","":"downstairs","session_id":"9110d401-07dc-4a95-af32-675b9320428e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:49.991285731Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1165,"":"io task","client":"2","":"downstairs","session_id":"9110d401-07dc-4a95-af32-675b9320428e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:49.991295691Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1165,"":"io task","client":"0","":"downstairs","session_id":"9110d401-07dc-4a95-af32-675b9320428e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:49.991305611Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1165,"":"io task","client":"1","":"downstairs","session_id":"9110d401-07dc-4a95-af32-675b9320428e"} {"msg":"ds_connection connected from Ok(127.0.0.1:39459)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:49.991357881Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1165,"":"io task","client":"1","":"downstairs","session_id":"9110d401-07dc-4a95-af32-675b9320428e"} {"msg":"ds_connection connected from Ok(127.0.0.1:58908)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:49.991369111Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1165,"":"io task","client":"2","":"downstairs","session_id":"9110d401-07dc-4a95-af32-675b9320428e"} {"msg":"ds_connection connected from Ok(127.0.0.1:52145)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:49.991379351Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1165,"":"io task","client":"0","":"downstairs","session_id":"9110d401-07dc-4a95-af32-675b9320428e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:49.991982022Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1165,"client":"1","":"downstairs","session_id":"9110d401-07dc-4a95-af32-675b9320428e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:49.992029732Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1165,"client":"2","":"downstairs","session_id":"9110d401-07dc-4a95-af32-675b9320428e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:49.992041752Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1165,"client":"0","":"downstairs","session_id":"9110d401-07dc-4a95-af32-675b9320428e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:49.992260722Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1165,"session_id":"9110d401-07dc-4a95-af32-675b9320428e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:49.992285362Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1165,"session_id":"9110d401-07dc-4a95-af32-675b9320428e"} {"msg":"[0]R flush_numbers[0..12]: [4, 7, 7, 5, 1, 8, 8, 5, 1, 8, 6, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:49.992366982Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1165,"":"downstairs","":"downstairs","session_id":"9110d401-07dc-4a95-af32-675b9320428e"} {"msg":"[0]R generation[0..12]: [2, 5, 5, 3, 1, 6, 6, 3, 1, 6, 4, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:49.992377912Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1165,"":"downstairs","":"downstairs","session_id":"9110d401-07dc-4a95-af32-675b9320428e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:49.992388972Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1165,"":"downstairs","":"downstairs","session_id":"9110d401-07dc-4a95-af32-675b9320428e"} {"msg":"[1]R flush_numbers[0..12]: [4, 7, 7, 5, 1, 8, 8, 5, 1, 8, 6, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:49.992401862Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1165,"":"downstairs","":"downstairs","session_id":"9110d401-07dc-4a95-af32-675b9320428e"} {"msg":"[1]R generation[0..12]: [2, 5, 5, 3, 1, 6, 6, 3, 1, 6, 4, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:49.992412022Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1165,"":"downstairs","":"downstairs","session_id":"9110d401-07dc-4a95-af32-675b9320428e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:49.992423022Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1165,"":"downstairs","":"downstairs","session_id":"9110d401-07dc-4a95-af32-675b9320428e"} {"msg":"[2]R flush_numbers[0..12]: [4, 7, 7, 5, 1, 8, 8, 5, 1, 8, 6, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:49.992433302Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1165,"":"downstairs","":"downstairs","session_id":"9110d401-07dc-4a95-af32-675b9320428e"} {"msg":"[2]R generation[0..12]: [2, 5, 5, 3, 1, 6, 6, 3, 1, 6, 4, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:49.992443392Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1165,"":"downstairs","":"downstairs","session_id":"9110d401-07dc-4a95-af32-675b9320428e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:49.992454422Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1165,"":"downstairs","":"downstairs","session_id":"9110d401-07dc-4a95-af32-675b9320428e"} {"msg":"Max found gen is 7","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:49.992463962Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1165,"":"downstairs","":"downstairs","session_id":"9110d401-07dc-4a95-af32-675b9320428e"} {"msg":"Generation requested: 8 >= found:7","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:49.992473492Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1165,"":"downstairs","":"downstairs","session_id":"9110d401-07dc-4a95-af32-675b9320428e"} {"msg":"Next flush: 9","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:49.992482932Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1165,"":"downstairs","":"downstairs","session_id":"9110d401-07dc-4a95-af32-675b9320428e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:49.992492422Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1165,"":"downstairs","":"downstairs","session_id":"9110d401-07dc-4a95-af32-675b9320428e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:49.992502322Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1165,"session_id":"9110d401-07dc-4a95-af32-675b9320428e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:49.992511092Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1165,"client":"0","":"downstairs","session_id":"9110d401-07dc-4a95-af32-675b9320428e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:49.992520892Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1165,"client":"1","":"downstairs","session_id":"9110d401-07dc-4a95-af32-675b9320428e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:49.992530572Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1165,"client":"2","":"downstairs","session_id":"9110d401-07dc-4a95-af32-675b9320428e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:49.992540452Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1165,"session_id":"9110d401-07dc-4a95-af32-675b9320428e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:49.992551062Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1165,"session_id":"9110d401-07dc-4a95-af32-675b9320428e"} {"msg":"c68a2df0-7cd9-413e-9c13-0addc2ecdfc6 is now active with session: 9110d401-07dc-4a95-af32-675b9320428e","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:49.992561102Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1165,"session_id":"9110d401-07dc-4a95-af32-675b9320428e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:49.992569943Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1165,"session_id":"9110d401-07dc-4a95-af32-675b9320428e"} {"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:49.992579233Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1165} {"msg":"Activated sub_volume c68a2df0-7cd9-413e-9c13-0addc2ecdfc6","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:49.992587343Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1165} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:49.992595473Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1165,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:49.992856183Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1165,"task":"crutest"} 01/30 Write block 50 len 8192 data: 3 3 02/30 Flush 03/30 Write block 61 len 28672 data: 3 3 3 3 3 2 2 04/30 Read block 132 len 40960 05/30 Read block 51 len 32768 06/30 Read block 149 len 28672 07/30 Read block 491 len 40960 08/30 Read block 348 len 28672 09/30 Write block 353 len 36864 data: 2 2 2 2 2 3 4 4 4 10/30 Read block 272 len 8192 11/30 Write block 567 len 32768 data: 3 3 3 4 3 3 2 2 12/30 Write block 463 len 36864 data: 2 2 2 3 3 3 3 2 2 13/30 Read block 214 len 32768 14/30 Read block 43 len 24576 15/30 Read block 435 len 32768 16/30 Write block 228 len 24576 data: 2 2 2 2 2 2 17/30 Write block 345 len 16384 data: 2 2 2 2 18/30 Read block 234 len 16384 19/30 Flush 20/30 Write block 558 len 4096 data: 2 21/30 Write block 276 len 8192 data: 4 4 22/30 Read block 167 len 4096 23/30 Read block 254 len 36864 24/30 Write block 537 len 24576 data: 2 2 2 2 2 2 25/30 Read block 98 len 12288 26/30 Read block 197 len 36864 27/30 Write block 572 len 4096 data: 4 28/30 Write block 233 len 40960 data: 3 2 2 2 2 2 2 2 2 2 29/30 Read block 586 len 20480 30/30 Read block 437 len 8192 ---------------------------------------------------------------- Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 7 Done Done Sent false 1003 Acked Read 10 Done Done Sent false 1004 Acked Read 8 Done Done Sent false 1005 Acked Read 7 Done Done Sent false 1006 Acked Read 10 Done Done Sent false 1007 Acked Read 7 Done Done Sent false 1008 Acked Write 9 Done Done Sent false 1009 Acked Read 2 Done Done Sent false 1010 Acked Write 8 Done Done Sent false 1011 Acked Write 9 Done Done Sent false 1012 Acked Read 8 Done Done Sent false 1013 Acked Read 6 Done Done Sent false 1014 Acked Read 8 Done Done Sent false 1015 Acked Write 6 Done Done Sent false 1016 Acked Write 4 Done Done Sent false 1017 Acked Read 4 Done Done Sent false 1018 Acked Flush 0 Done Done Sent false 1019 Acked Write 1 Done Done Sent false 1020 Acked Write 2 Done Done Sent false 1021 Acked Read 1 Done Done Sent false 1022 Acked Read 9 Done Done Sent false 1023 Acked Write 6 Done Done Sent false 1024 Acked Read 3 Done Done Sent false 1025 Acked Read 9 Done Done Sent false 1026 Acked Write 1 Done Done Sent false 1027 Acked Write 10 Done Done Sent false 1028 Acked Read 5 Done Done Sent false 1029 Acked Read 2 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 28 29 Done 28 27 0 55 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) Some(JobId(1001)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:19:50.001752386Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1165} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:50.001923327Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1165,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 4 4 4 F F F 1 020-039 5 5 5 7 7 7 F F F 2 040-059 8 8 8 9 9 9 F F F 3 060-079 8 8 3 10 10 5 F F F <--- 4 080-099 1 1 1 1 1 1 F F F 5 100-119 6 6 6 8 8 8 F F F 6 120-139 6 6 6 8 8 8 F F F 7 140-159 3 3 3 5 5 5 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 6 6 6 8 8 8 F F F 10 200-219 4 4 4 6 6 6 F F F 11 220-239 8 8 1 10 10 1 T T F <--- 12 240-259 6 6 6 8 8 8 T T F <--- 13 260-279 6 6 6 8 8 8 T T F <--- 14 280-299 4 4 4 6 6 6 F F F 15 300-319 5 5 5 7 7 7 F F F 16 320-339 5 5 5 7 7 7 F F F 17 340-359 8 8 2 10 10 4 F F F <--- 18 360-379 8 8 2 10 10 4 F F F <--- 19 380-399 6 6 6 8 8 8 F F F 20 400-419 4 4 4 6 6 6 F F F 21 420-439 6 6 6 8 8 8 F F F 22 440-459 2 2 2 4 4 4 F F F 23 460-479 8 8 3 10 10 5 F F F <--- 24 480-499 2 2 2 3 3 3 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 1 1 1 1 1 1 T T F <--- 27 540-559 1 1 1 1 1 1 T T F <--- 28 560-579 8 8 6 10 10 8 T T F <--- 29 580-599 1 1 1 1 1 1 F F F Max gen: 8, Max flush: 10 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 4 Verifying data now No change after verify Loop: 4 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 4 4 4 F F F 1 020-039 5 5 5 7 7 7 F F F 2 040-059 8 8 8 9 9 9 F F F 3 060-079 8 8 8 10 10 10 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 6 6 6 8 8 8 F F F 6 120-139 6 6 6 8 8 8 F F F 7 140-159 3 3 3 5 5 5 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 6 6 6 8 8 8 F F F 10 200-219 4 4 4 6 6 6 F F F 11 220-239 9 9 9 11 11 11 F F F 12 240-259 9 9 9 11 11 11 F F F 13 260-279 9 9 9 11 11 11 F F F 14 280-299 4 4 4 6 6 6 F F F 15 300-319 5 5 5 7 7 7 F F F 16 320-339 5 5 5 7 7 7 F F F 17 340-359 8 8 8 10 10 10 F F F 18 360-379 8 8 8 10 10 10 F F F 19 380-399 6 6 6 8 8 8 F F F 20 400-419 4 4 4 6 6 6 F F F 21 420-439 6 6 6 8 8 8 F F F 22 440-459 2 2 2 4 4 4 F F F 23 460-479 8 8 8 10 10 10 F F F 24 480-499 2 2 2 3 3 3 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 9 9 9 11 11 11 F F F 27 540-559 9 9 9 11 11 11 F F F 28 560-579 9 9 9 11 11 11 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 9, Max flush: 11 Begin loop 5 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:54.237873309Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1173,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:19:54.24500562Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1173,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:54.24506943Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1173,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:54.25211118Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1173,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:54.252379661Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1173} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:54.252404991Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1173} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:54.252413581Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1173} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:54.252425351Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1173} {"msg":"Crucible 6c28dfd8-f684-4232-a623-54198a840e10 has session id: 84d86b3a-2eae-40a1-be6c-5f7c5e4426d6","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:54.252765201Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1173,"session_id":"84d86b3a-2eae-40a1-be6c-5f7c5e4426d6"} {"msg":"Upstairs opts: Upstairs UUID: 6c28dfd8-f684-4232-a623-54198a840e10, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:54.252829431Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1173,"session_id":"84d86b3a-2eae-40a1-be6c-5f7c5e4426d6"} {"msg":"Crucible stats registered with UUID: 6c28dfd8-f684-4232-a623-54198a840e10","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:54.252843841Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1173,"session_id":"84d86b3a-2eae-40a1-be6c-5f7c5e4426d6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:19:54.252853271Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1173,"session_id":"84d86b3a-2eae-40a1-be6c-5f7c5e4426d6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:54.252983272Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1173,"":"io task","client":"1","":"downstairs","session_id":"84d86b3a-2eae-40a1-be6c-5f7c5e4426d6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:54.253015392Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1173,"":"io task","client":"2","":"downstairs","session_id":"84d86b3a-2eae-40a1-be6c-5f7c5e4426d6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:54.253026462Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1173,"":"io task","client":"0","":"downstairs","session_id":"84d86b3a-2eae-40a1-be6c-5f7c5e4426d6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:54.253173132Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1173,"task":"crutest"} {"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:54.253190782Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1173} {"msg":"6c28dfd8-f684-4232-a623-54198a840e10 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:54.253233152Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1173,"session_id":"84d86b3a-2eae-40a1-be6c-5f7c5e4426d6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:54.253242692Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1173,"client":"0","":"downstairs","session_id":"84d86b3a-2eae-40a1-be6c-5f7c5e4426d6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:54.253252342Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1173,"client":"1","":"downstairs","session_id":"84d86b3a-2eae-40a1-be6c-5f7c5e4426d6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:54.253262032Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1173,"client":"2","":"downstairs","session_id":"84d86b3a-2eae-40a1-be6c-5f7c5e4426d6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:54.253271672Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1173,"":"io task","client":"2","":"downstairs","session_id":"84d86b3a-2eae-40a1-be6c-5f7c5e4426d6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:54.253281862Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1173,"":"io task","client":"0","":"downstairs","session_id":"84d86b3a-2eae-40a1-be6c-5f7c5e4426d6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:54.253291962Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1173,"":"io task","client":"1","":"downstairs","session_id":"84d86b3a-2eae-40a1-be6c-5f7c5e4426d6"} {"msg":"ds_connection connected from Ok(127.0.0.1:53014)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:54.253338852Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1173,"":"io task","client":"1","":"downstairs","session_id":"84d86b3a-2eae-40a1-be6c-5f7c5e4426d6"} {"msg":"ds_connection connected from Ok(127.0.0.1:59394)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:54.253349852Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1173,"":"io task","client":"0","":"downstairs","session_id":"84d86b3a-2eae-40a1-be6c-5f7c5e4426d6"} {"msg":"ds_connection connected from Ok(127.0.0.1:54870)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:54.253362512Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1173,"":"io task","client":"2","":"downstairs","session_id":"84d86b3a-2eae-40a1-be6c-5f7c5e4426d6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:54.254043613Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1173,"client":"2","":"downstairs","session_id":"84d86b3a-2eae-40a1-be6c-5f7c5e4426d6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:54.254062783Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1173,"client":"1","":"downstairs","session_id":"84d86b3a-2eae-40a1-be6c-5f7c5e4426d6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:54.254073863Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1173,"client":"0","":"downstairs","session_id":"84d86b3a-2eae-40a1-be6c-5f7c5e4426d6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:54.254363464Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1173,"session_id":"84d86b3a-2eae-40a1-be6c-5f7c5e4426d6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:54.254407784Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1173,"session_id":"84d86b3a-2eae-40a1-be6c-5f7c5e4426d6"} {"msg":"[0]R flush_numbers[0..12]: [4, 7, 9, 10, 1, 8, 8, 5, 1, 8, 6, 11]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:54.254450644Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1173,"":"downstairs","":"downstairs","session_id":"84d86b3a-2eae-40a1-be6c-5f7c5e4426d6"} {"msg":"[0]R generation[0..12]: [2, 5, 8, 8, 1, 6, 6, 3, 1, 6, 4, 9]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:54.254461574Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1173,"":"downstairs","":"downstairs","session_id":"84d86b3a-2eae-40a1-be6c-5f7c5e4426d6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:54.254472814Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1173,"":"downstairs","":"downstairs","session_id":"84d86b3a-2eae-40a1-be6c-5f7c5e4426d6"} {"msg":"[1]R flush_numbers[0..12]: [4, 7, 9, 10, 1, 8, 8, 5, 1, 8, 6, 11]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:54.254483094Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1173,"":"downstairs","":"downstairs","session_id":"84d86b3a-2eae-40a1-be6c-5f7c5e4426d6"} {"msg":"[1]R generation[0..12]: [2, 5, 8, 8, 1, 6, 6, 3, 1, 6, 4, 9]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:54.254493494Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1173,"":"downstairs","":"downstairs","session_id":"84d86b3a-2eae-40a1-be6c-5f7c5e4426d6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:54.254504594Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1173,"":"downstairs","":"downstairs","session_id":"84d86b3a-2eae-40a1-be6c-5f7c5e4426d6"} {"msg":"[2]R flush_numbers[0..12]: [4, 7, 9, 10, 1, 8, 8, 5, 1, 8, 6, 11]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:54.254514834Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1173,"":"downstairs","":"downstairs","session_id":"84d86b3a-2eae-40a1-be6c-5f7c5e4426d6"} {"msg":"[2]R generation[0..12]: [2, 5, 8, 8, 1, 6, 6, 3, 1, 6, 4, 9]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:54.254525064Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1173,"":"downstairs","":"downstairs","session_id":"84d86b3a-2eae-40a1-be6c-5f7c5e4426d6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:54.254538354Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1173,"":"downstairs","":"downstairs","session_id":"84d86b3a-2eae-40a1-be6c-5f7c5e4426d6"} {"msg":"Max found gen is 10","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:54.254547794Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1173,"":"downstairs","":"downstairs","session_id":"84d86b3a-2eae-40a1-be6c-5f7c5e4426d6"} {"msg":"Generation requested: 10 >= found:10","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:54.254557624Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1173,"":"downstairs","":"downstairs","session_id":"84d86b3a-2eae-40a1-be6c-5f7c5e4426d6"} {"msg":"Next flush: 12","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:54.254567144Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1173,"":"downstairs","":"downstairs","session_id":"84d86b3a-2eae-40a1-be6c-5f7c5e4426d6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:54.254576654Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1173,"":"downstairs","":"downstairs","session_id":"84d86b3a-2eae-40a1-be6c-5f7c5e4426d6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:54.254586634Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1173,"session_id":"84d86b3a-2eae-40a1-be6c-5f7c5e4426d6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:54.254595574Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1173,"client":"0","":"downstairs","session_id":"84d86b3a-2eae-40a1-be6c-5f7c5e4426d6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:54.254605484Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1173,"client":"1","":"downstairs","session_id":"84d86b3a-2eae-40a1-be6c-5f7c5e4426d6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:54.254615424Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1173,"client":"2","":"downstairs","session_id":"84d86b3a-2eae-40a1-be6c-5f7c5e4426d6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:54.254625724Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1173,"session_id":"84d86b3a-2eae-40a1-be6c-5f7c5e4426d6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:54.254635114Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1173,"session_id":"84d86b3a-2eae-40a1-be6c-5f7c5e4426d6"} {"msg":"6c28dfd8-f684-4232-a623-54198a840e10 is now active with session: 84d86b3a-2eae-40a1-be6c-5f7c5e4426d6","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:54.254645194Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1173,"session_id":"84d86b3a-2eae-40a1-be6c-5f7c5e4426d6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:54.254654014Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1173,"session_id":"84d86b3a-2eae-40a1-be6c-5f7c5e4426d6"} {"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:54.254663064Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1173} {"msg":"Activated sub_volume 6c28dfd8-f684-4232-a623-54198a840e10","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:54.254671164Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1173} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:54.254679404Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1173,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:54.254986125Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1173,"task":"crutest"} 01/30 Write block 62 len 20480 data: 4 4 4 4 3 02/30 Read block 265 len 4096 03/30 Write block 375 len 16384 data: 2 2 2 2 04/30 Write block 201 len 20480 data: 2 2 2 2 2 05/30 Write block 18 len 40960 data: 2 2 2 2 2 2 2 2 2 2 06/30 Write block 476 len 20480 data: 2 2 2 2 2 07/30 Flush 08/30 Write block 76 len 20480 data: 2 2 2 2 2 09/30 Write block 16 len 40960 data: 2 2 3 3 3 3 3 3 3 3 10/30 Flush 11/30 Write block 337 len 4096 data: 2 12/30 Read block 576 len 32768 13/30 Write block 388 len 32768 data: 3 3 3 3 3 2 3 3 14/30 Write block 135 len 28672 data: 2 2 2 2 2 2 2 15/30 Write block 191 len 40960 data: 2 2 3 2 2 3 3 3 2 2 16/30 Read block 306 len 4096 17/30 Write block 372 len 40960 data: 2 2 2 3 3 3 3 2 2 2 18/30 Read block 389 len 40960 19/30 Read block 204 len 12288 20/30 Write block 322 len 36864 data: 3 4 3 3 3 3 3 3 3 21/30 Read block 221 len 24576 22/30 Read block 246 len 24576 23/30 Read block 445 len 16384 24/30 Read block 564 len 16384 25/30 Write block 20 len 20480 data: 4 4 4 4 4 26/30 Flush 27/30 Write block 179 len 12288 data: 2 2 2 28/30 Read block 335 len 24576 29/30 Read block 374 len 24576 30/30 Read block 87 len 8192 ---------------------------------------------------------------- Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Read 1 Done Done Done false 1002 Acked Write 4 Done Done Done false 1003 Acked Write 5 Sent Done Done false 1004 Acked Write 10 Sent Done Done false 1005 Acked Write 5 Sent Done Done false 1006 Acked Flush 0 Sent Done Done false 1007 Acked Write 5 Sent Done Done false 1008 Acked Write 10 Sent Done Done false 1009 Acked Flush 0 Sent Done Done false 1010 Acked Write 1 Sent Done Done false 1011 Acked Read 8 Sent Done Done false 1012 Acked Write 8 Sent Done Done false 1013 Acked Write 7 Sent Done Done false 1014 Acked Write 10 Sent Done Done false 1015 Acked Read 1 Sent Done Done false 1016 Acked Write 10 Sent Done Done false 1017 Acked Read 10 Sent Done Done false 1018 Acked Read 3 Sent Done Done false 1019 Acked Write 9 Sent Done Done false 1020 Acked Read 6 Sent Done Done false 1021 Acked Read 6 Sent Done Done false 1022 Acked Read 4 Sent Done Done false 1023 Acked Read 4 Sent Done Done false 1024 Acked Write 5 Sent Done Done false 1025 Acked Flush 0 Sent Done Done false 1026 Acked Write 3 Sent Done Done false 1027 Acked Read 6 Sent Done Done false 1028 Acked Read 6 Sent Done Done false 1029 Acked Read 2 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 27 0 0 27 Done 3 30 30 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:19:54.263773288Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1173} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:54.263940668Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1173,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 10 10 4 13 13 F F F <--- 1 020-039 5 10 10 7 14 14 F F F <--- 2 040-059 8 8 8 9 9 9 F F F 3 060-079 8 10 10 10 13 13 T F F <--- 4 080-099 1 10 10 1 13 13 F F F <--- 5 100-119 6 6 6 8 8 8 F F F 6 120-139 6 10 10 8 14 14 F F F <--- 7 140-159 3 10 10 5 14 14 F F F <--- 8 160-179 1 1 1 1 1 1 F T T <--- 9 180-199 6 10 10 8 14 14 F T T <--- 10 200-219 4 10 10 6 14 14 F F F <--- 11 220-239 9 9 9 11 11 11 F F F 12 240-259 9 9 9 11 11 11 F F F 13 260-279 9 9 9 11 11 11 F F F 14 280-299 4 4 4 6 6 6 F F F 15 300-319 5 5 5 7 7 7 F F F 16 320-339 5 10 10 7 14 14 F F F <--- 17 340-359 8 8 8 10 10 10 F F F 18 360-379 8 10 10 10 14 14 T F F <--- 19 380-399 6 10 10 8 14 14 F F F <--- 20 400-419 4 4 4 6 6 6 F F F 21 420-439 6 6 6 8 8 8 F F F 22 440-459 2 2 2 4 4 4 F F F 23 460-479 8 10 10 10 12 12 F F F <--- 24 480-499 2 10 10 3 12 12 F F F <--- 25 500-519 1 1 1 1 1 1 F F F 26 520-539 9 9 9 11 11 11 F F F 27 540-559 9 9 9 11 11 11 F F F 28 560-579 9 9 9 11 11 11 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 10, Max flush: 14 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 5 Verifying data now No change after verify Loop: 5 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 13 13 13 F F F 1 020-039 10 10 10 14 14 14 F F F 2 040-059 8 8 8 9 9 9 F F F 3 060-079 10 10 10 13 13 13 F F F 4 080-099 10 10 10 13 13 13 F F F 5 100-119 6 6 6 8 8 8 F F F 6 120-139 10 10 10 14 14 14 F F F 7 140-159 10 10 10 14 14 14 F F F 8 160-179 11 11 11 15 15 15 F F F 9 180-199 11 11 11 15 15 15 F F F 10 200-219 10 10 10 14 14 14 F F F 11 220-239 9 9 9 11 11 11 F F F 12 240-259 9 9 9 11 11 11 F F F 13 260-279 9 9 9 11 11 11 F F F 14 280-299 4 4 4 6 6 6 F F F 15 300-319 5 5 5 7 7 7 F F F 16 320-339 10 10 10 14 14 14 F F F 17 340-359 8 8 8 10 10 10 F F F 18 360-379 10 10 10 14 14 14 F F F 19 380-399 10 10 10 14 14 14 F F F 20 400-419 4 4 4 6 6 6 F F F 21 420-439 6 6 6 8 8 8 F F F 22 440-459 2 2 2 4 4 4 F F F 23 460-479 10 10 10 12 12 12 F F F 24 480-499 10 10 10 12 12 12 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 9 9 9 11 11 11 F F F 27 540-559 9 9 9 11 11 11 F F F 28 560-579 9 9 9 11 11 11 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 11, Max flush: 15 Begin loop 6 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:58.514510382Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1181,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:19:58.521728193Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1181,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:58.521790113Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1181,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:58.528931654Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1181,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:58.529230185Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1181} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:58.529257405Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1181} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:58.529266165Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1181} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:58.529278055Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1181} {"msg":"Crucible 8b3073ac-cf32-4754-8ef0-a75aaab247aa has session id: 7b27cacb-f80d-4d37-8d67-6b5dd282fa17","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:58.529636425Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1181,"session_id":"7b27cacb-f80d-4d37-8d67-6b5dd282fa17"} {"msg":"Upstairs opts: Upstairs UUID: 8b3073ac-cf32-4754-8ef0-a75aaab247aa, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:58.529659545Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1181,"session_id":"7b27cacb-f80d-4d37-8d67-6b5dd282fa17"} {"msg":"Crucible stats registered with UUID: 8b3073ac-cf32-4754-8ef0-a75aaab247aa","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:58.529669485Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1181,"session_id":"7b27cacb-f80d-4d37-8d67-6b5dd282fa17"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:19:58.529678735Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1181,"session_id":"7b27cacb-f80d-4d37-8d67-6b5dd282fa17"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:58.529828135Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1181,"":"io task","client":"2","":"downstairs","session_id":"7b27cacb-f80d-4d37-8d67-6b5dd282fa17"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:58.529845596Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1181,"":"io task","client":"0","":"downstairs","session_id":"7b27cacb-f80d-4d37-8d67-6b5dd282fa17"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:58.529858076Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1181,"":"io task","client":"1","":"downstairs","session_id":"7b27cacb-f80d-4d37-8d67-6b5dd282fa17"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:58.530013046Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1181,"task":"crutest"} {"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:58.530028846Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1181} {"msg":"8b3073ac-cf32-4754-8ef0-a75aaab247aa active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:58.530064216Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1181,"session_id":"7b27cacb-f80d-4d37-8d67-6b5dd282fa17"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:58.530073696Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1181,"client":"0","":"downstairs","session_id":"7b27cacb-f80d-4d37-8d67-6b5dd282fa17"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:58.530083496Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1181,"client":"1","":"downstairs","session_id":"7b27cacb-f80d-4d37-8d67-6b5dd282fa17"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:58.530093316Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1181,"client":"2","":"downstairs","session_id":"7b27cacb-f80d-4d37-8d67-6b5dd282fa17"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:58.530103076Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1181,"":"io task","client":"2","":"downstairs","session_id":"7b27cacb-f80d-4d37-8d67-6b5dd282fa17"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:58.530113096Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1181,"":"io task","client":"0","":"downstairs","session_id":"7b27cacb-f80d-4d37-8d67-6b5dd282fa17"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:58.530123286Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1181,"":"io task","client":"1","":"downstairs","session_id":"7b27cacb-f80d-4d37-8d67-6b5dd282fa17"} {"msg":"ds_connection connected from Ok(127.0.0.1:52031)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:58.530233306Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1181,"":"io task","client":"2","":"downstairs","session_id":"7b27cacb-f80d-4d37-8d67-6b5dd282fa17"} {"msg":"ds_connection connected from Ok(127.0.0.1:50581)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:58.530259026Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1181,"":"io task","client":"1","":"downstairs","session_id":"7b27cacb-f80d-4d37-8d67-6b5dd282fa17"} {"msg":"ds_connection connected from Ok(127.0.0.1:64916)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:58.530270706Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1181,"":"io task","client":"0","":"downstairs","session_id":"7b27cacb-f80d-4d37-8d67-6b5dd282fa17"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:58.530977287Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1181,"client":"1","":"downstairs","session_id":"7b27cacb-f80d-4d37-8d67-6b5dd282fa17"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:58.530995727Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1181,"client":"0","":"downstairs","session_id":"7b27cacb-f80d-4d37-8d67-6b5dd282fa17"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:58.531009217Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1181,"client":"2","":"downstairs","session_id":"7b27cacb-f80d-4d37-8d67-6b5dd282fa17"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:58.531316408Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1181,"session_id":"7b27cacb-f80d-4d37-8d67-6b5dd282fa17"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:58.531326128Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1181,"session_id":"7b27cacb-f80d-4d37-8d67-6b5dd282fa17"} {"msg":"[0]R flush_numbers[0..12]: [13, 14, 9, 13, 13, 8, 14, 14, 15, 15, 14, 11]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:58.531365828Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1181,"":"downstairs","":"downstairs","session_id":"7b27cacb-f80d-4d37-8d67-6b5dd282fa17"} {"msg":"[0]R generation[0..12]: [10, 10, 8, 10, 10, 6, 10, 10, 11, 11, 10, 9]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:58.531376908Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1181,"":"downstairs","":"downstairs","session_id":"7b27cacb-f80d-4d37-8d67-6b5dd282fa17"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:58.531388108Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1181,"":"downstairs","":"downstairs","session_id":"7b27cacb-f80d-4d37-8d67-6b5dd282fa17"} {"msg":"[1]R flush_numbers[0..12]: [13, 14, 9, 13, 13, 8, 14, 14, 15, 15, 14, 11]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:58.531398658Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1181,"":"downstairs","":"downstairs","session_id":"7b27cacb-f80d-4d37-8d67-6b5dd282fa17"} {"msg":"[1]R generation[0..12]: [10, 10, 8, 10, 10, 6, 10, 10, 11, 11, 10, 9]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:58.531409018Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1181,"":"downstairs","":"downstairs","session_id":"7b27cacb-f80d-4d37-8d67-6b5dd282fa17"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:58.531420008Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1181,"":"downstairs","":"downstairs","session_id":"7b27cacb-f80d-4d37-8d67-6b5dd282fa17"} {"msg":"[2]R flush_numbers[0..12]: [13, 14, 9, 13, 13, 8, 14, 14, 15, 15, 14, 11]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:58.531430548Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1181,"":"downstairs","":"downstairs","session_id":"7b27cacb-f80d-4d37-8d67-6b5dd282fa17"} {"msg":"[2]R generation[0..12]: [10, 10, 8, 10, 10, 6, 10, 10, 11, 11, 10, 9]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:58.531441048Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1181,"":"downstairs","":"downstairs","session_id":"7b27cacb-f80d-4d37-8d67-6b5dd282fa17"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:58.531452188Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1181,"":"downstairs","":"downstairs","session_id":"7b27cacb-f80d-4d37-8d67-6b5dd282fa17"} {"msg":"Max found gen is 12","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:58.531461648Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1181,"":"downstairs","":"downstairs","session_id":"7b27cacb-f80d-4d37-8d67-6b5dd282fa17"} {"msg":"Generation requested: 12 >= found:12","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:58.531471488Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1181,"":"downstairs","":"downstairs","session_id":"7b27cacb-f80d-4d37-8d67-6b5dd282fa17"} {"msg":"Next flush: 16","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:58.531480788Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1181,"":"downstairs","":"downstairs","session_id":"7b27cacb-f80d-4d37-8d67-6b5dd282fa17"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:58.531491948Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1181,"":"downstairs","":"downstairs","session_id":"7b27cacb-f80d-4d37-8d67-6b5dd282fa17"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:58.531501778Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1181,"session_id":"7b27cacb-f80d-4d37-8d67-6b5dd282fa17"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:58.531510698Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1181,"client":"0","":"downstairs","session_id":"7b27cacb-f80d-4d37-8d67-6b5dd282fa17"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:58.531520378Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1181,"client":"1","":"downstairs","session_id":"7b27cacb-f80d-4d37-8d67-6b5dd282fa17"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:58.531530118Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1181,"client":"2","":"downstairs","session_id":"7b27cacb-f80d-4d37-8d67-6b5dd282fa17"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:58.531540078Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1181,"session_id":"7b27cacb-f80d-4d37-8d67-6b5dd282fa17"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:58.531549388Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1181,"session_id":"7b27cacb-f80d-4d37-8d67-6b5dd282fa17"} {"msg":"8b3073ac-cf32-4754-8ef0-a75aaab247aa is now active with session: 7b27cacb-f80d-4d37-8d67-6b5dd282fa17","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:58.531559398Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1181,"session_id":"7b27cacb-f80d-4d37-8d67-6b5dd282fa17"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:58.531568208Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1181,"session_id":"7b27cacb-f80d-4d37-8d67-6b5dd282fa17"} {"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:58.531577338Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1181} {"msg":"Activated sub_volume 8b3073ac-cf32-4754-8ef0-a75aaab247aa","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:58.531585628Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1181} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:58.531593968Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1181,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:58.531898749Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1181,"task":"crutest"} 01/30 Write block 412 len 36864 data: 2 2 2 2 2 3 3 3 3 02/30 Write block 346 len 16384 data: 3 3 3 2 03/30 Write block 552 len 8192 data: 2 2 04/30 Write block 527 len 12288 data: 2 2 2 05/30 Write block 539 len 4096 data: 3 06/30 Write block 512 len 12288 data: 2 2 2 07/30 Read block 581 len 20480 08/30 Read block 198 len 16384 09/30 Write block 150 len 28672 data: 3 3 3 3 2 2 2 10/30 Write block 412 len 24576 data: 3 3 3 3 3 4 11/30 Read block 238 len 32768 12/30 Write block 191 len 8192 data: 3 3 13/30 Write block 489 len 4096 data: 3 14/30 Write block 370 len 32768 data: 2 2 3 3 3 4 4 4 15/30 Write block 335 len 4096 data: 2 16/30 Write block 346 len 8192 data: 4 4 17/30 Read block 337 len 40960 18/30 Read block 382 len 32768 19/30 Write block 406 len 8192 data: 2 2 20/30 Write block 523 len 40960 data: 2 2 2 2 3 3 3 2 2 2 21/30 Read block 39 len 24576 22/30 Read block 86 len 32768 23/30 Read block 389 len 8192 24/30 Write block 334 len 28672 data: 2 3 2 3 2 2 2 25/30 Write block 574 len 24576 data: 3 2 2 3 3 2 26/30 Read block 365 len 36864 27/30 Read block 177 len 24576 28/30 Write block 569 len 4096 data: 4 29/30 Write block 484 len 20480 data: 2 2 2 2 2 30/30 Write block 320 len 24576 data: 2 2 4 5 4 4 ---------------------------------------------------------------- Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Write 4 Sent Done Done false 1002 Acked Write 2 Sent Done Done false 1003 Acked Write 3 Sent Done Done false 1004 Acked Write 1 Sent Done Done false 1005 Acked Write 3 Sent Done Done false 1006 Acked Read 5 Sent Done Done false 1007 Acked Read 4 Sent Done Done false 1008 Acked Write 7 Sent Done Done false 1009 Acked Write 6 Sent Done Done false 1010 Acked Read 8 Sent Done Done false 1011 Acked Write 2 Sent Done Done false 1012 Acked Write 1 Sent Done Done false 1013 Acked Write 8 Sent Done Done false 1014 Acked Write 1 Sent Done Done false 1015 Acked Write 2 Sent Done Done false 1016 Acked Read 10 Sent Done Done false 1017 Acked Read 8 Sent Done Done false 1018 Acked Write 2 Sent Done Done false 1019 Acked Write 10 Sent Done Done false 1020 Acked Read 6 Sent Done Done false 1021 Acked Read 8 Sent Done Done false 1022 Acked Read 2 Sent Done Done false 1023 Acked Write 7 Sent Done Done false 1024 Acked Write 6 Sent Done Done false 1025 Acked Read 9 Sent Done Done false 1026 Acked Read 6 Sent Done Done false 1027 Acked Write 1 Sent Done Done false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 2 2 33 Done 1 28 28 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:19:58.53937791Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1181} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:19:58.53957223Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1181,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 13 13 13 F F F 1 020-039 10 10 10 14 14 14 F F F 2 040-059 8 8 8 9 9 9 F F F 3 060-079 10 10 10 13 13 13 F F F 4 080-099 10 10 10 13 13 13 F F F 5 100-119 6 6 6 8 8 8 F F F 6 120-139 10 10 10 14 14 14 F F F 7 140-159 10 10 10 14 14 14 F T T <--- 8 160-179 11 11 11 15 15 15 F F F 9 180-199 11 11 11 15 15 15 F T T <--- 10 200-219 10 10 10 14 14 14 F F F 11 220-239 9 9 9 11 11 11 F F F 12 240-259 9 9 9 11 11 11 F F F 13 260-279 9 9 9 11 11 11 F F F 14 280-299 4 4 4 6 6 6 F F F 15 300-319 5 5 5 7 7 7 F F F 16 320-339 10 10 10 14 14 14 F T T <--- 17 340-359 8 8 8 10 10 10 F T T <--- 18 360-379 10 10 10 14 14 14 F T T <--- 19 380-399 10 10 10 14 14 14 F F F 20 400-419 4 4 4 6 6 6 T T T 21 420-439 6 6 6 8 8 8 T T T 22 440-459 2 2 2 4 4 4 F F F 23 460-479 10 10 10 12 12 12 F F F 24 480-499 10 10 10 12 12 12 F T T <--- 25 500-519 1 1 1 1 1 1 F T T <--- 26 520-539 9 9 9 11 11 11 F T T <--- 27 540-559 9 9 9 11 11 11 F T T <--- 28 560-579 9 9 9 11 11 11 F T T <--- 29 580-599 1 1 1 1 1 1 F F F Max gen: 11, Max flush: 15 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 6 Verifying data now Files /tmp/test_repair/test_repair_verify.data and /tmp/test_repair/test_repair_verify.data.last differ 408a409,410 > 2, > 2, 413,419d414 < 1, < 1, < 2, < 2, < 2, < 2, < 2, 422a418,422 > 3, > 3, > 4, > 3, > 3, diff found after verify Loop: 6 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 13 13 13 F F F 1 020-039 10 10 10 14 14 14 F F F 2 040-059 8 8 8 9 9 9 F F F 3 060-079 10 10 10 13 13 13 F F F 4 080-099 10 10 10 13 13 13 F F F 5 100-119 6 6 6 8 8 8 F F F 6 120-139 10 10 10 14 14 14 F F F 7 140-159 12 12 12 16 16 16 F F F 8 160-179 11 11 11 15 15 15 F F F 9 180-199 12 12 12 16 16 16 F F F 10 200-219 10 10 10 14 14 14 F F F 11 220-239 9 9 9 11 11 11 F F F 12 240-259 9 9 9 11 11 11 F F F 13 260-279 9 9 9 11 11 11 F F F 14 280-299 4 4 4 6 6 6 F F F 15 300-319 5 5 5 7 7 7 F F F 16 320-339 12 12 12 16 16 16 F F F 17 340-359 12 12 12 16 16 16 F F F 18 360-379 12 12 12 16 16 16 F F F 19 380-399 10 10 10 14 14 14 F F F 20 400-419 12 12 12 16 16 16 F F F 21 420-439 12 12 12 16 16 16 F F F 22 440-459 2 2 2 4 4 4 F F F 23 460-479 10 10 10 12 12 12 F F F 24 480-499 12 12 12 16 16 16 F F F 25 500-519 12 12 12 16 16 16 F F F 26 520-539 12 12 12 16 16 16 F F F 27 540-559 12 12 12 16 16 16 F F F 28 560-579 12 12 12 16 16 16 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 12, Max flush: 16 Begin loop 7 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:02.803255804Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1192,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:02.811438007Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1192,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:02.811735987Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1192} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:02.811763517Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1192} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:02.811772207Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1192} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:02.811784077Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1192} {"msg":"Crucible 65547ccf-f162-428f-864d-fbed82652e2c has session id: 8cf5390a-8ad2-4842-8029-e242f88b0682","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:02.812174708Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1192,"session_id":"8cf5390a-8ad2-4842-8029-e242f88b0682"} {"msg":"Upstairs opts: Upstairs UUID: 65547ccf-f162-428f-864d-fbed82652e2c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:02.812204188Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1192,"session_id":"8cf5390a-8ad2-4842-8029-e242f88b0682"} {"msg":"Crucible stats registered with UUID: 65547ccf-f162-428f-864d-fbed82652e2c","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:02.812215258Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1192,"session_id":"8cf5390a-8ad2-4842-8029-e242f88b0682"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:02.812224558Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1192,"session_id":"8cf5390a-8ad2-4842-8029-e242f88b0682"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:02.812348928Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1192,"":"io task","client":"0","":"downstairs","session_id":"8cf5390a-8ad2-4842-8029-e242f88b0682"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:02.812381208Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1192,"":"io task","client":"2","":"downstairs","session_id":"8cf5390a-8ad2-4842-8029-e242f88b0682"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:02.812392308Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1192,"":"io task","client":"1","":"downstairs","session_id":"8cf5390a-8ad2-4842-8029-e242f88b0682"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:02.812526828Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1192,"task":"crutest"} {"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:02.812545678Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1192} {"msg":"65547ccf-f162-428f-864d-fbed82652e2c active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:02.812569418Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1192,"session_id":"8cf5390a-8ad2-4842-8029-e242f88b0682"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:02.812578788Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1192,"client":"0","":"downstairs","session_id":"8cf5390a-8ad2-4842-8029-e242f88b0682"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:02.812590558Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1192,"client":"1","":"downstairs","session_id":"8cf5390a-8ad2-4842-8029-e242f88b0682"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:02.812600388Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1192,"client":"2","":"downstairs","session_id":"8cf5390a-8ad2-4842-8029-e242f88b0682"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:02.812610098Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1192,"":"io task","client":"2","":"downstairs","session_id":"8cf5390a-8ad2-4842-8029-e242f88b0682"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:02.812620148Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1192,"":"io task","client":"0","":"downstairs","session_id":"8cf5390a-8ad2-4842-8029-e242f88b0682"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:02.812630148Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1192,"":"io task","client":"1","":"downstairs","session_id":"8cf5390a-8ad2-4842-8029-e242f88b0682"} {"msg":"ds_connection connected from Ok(127.0.0.1:36088)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:02.812721789Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1192,"":"io task","client":"1","":"downstairs","session_id":"8cf5390a-8ad2-4842-8029-e242f88b0682"} {"msg":"ds_connection connected from Ok(127.0.0.1:61583)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:02.812747189Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1192,"":"io task","client":"0","":"downstairs","session_id":"8cf5390a-8ad2-4842-8029-e242f88b0682"} {"msg":"ds_connection connected from Ok(127.0.0.1:48538)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:02.812758859Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1192,"":"io task","client":"2","":"downstairs","session_id":"8cf5390a-8ad2-4842-8029-e242f88b0682"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:02.81343269Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1192,"client":"2","":"downstairs","session_id":"8cf5390a-8ad2-4842-8029-e242f88b0682"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:02.81344716Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1192,"client":"0","":"downstairs","session_id":"8cf5390a-8ad2-4842-8029-e242f88b0682"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:02.81349444Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1192,"client":"1","":"downstairs","session_id":"8cf5390a-8ad2-4842-8029-e242f88b0682"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:02.81376002Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1192,"session_id":"8cf5390a-8ad2-4842-8029-e242f88b0682"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:02.81378041Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1192,"session_id":"8cf5390a-8ad2-4842-8029-e242f88b0682"} {"msg":"[0]R flush_numbers[0..12]: [13, 14, 9, 13, 13, 8, 14, 16, 15, 16, 14, 11]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:02.81382865Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1192,"":"downstairs","":"downstairs","session_id":"8cf5390a-8ad2-4842-8029-e242f88b0682"} {"msg":"[0]R generation[0..12]: [10, 10, 8, 10, 10, 6, 10, 12, 11, 12, 10, 9]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:02.8138402Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1192,"":"downstairs","":"downstairs","session_id":"8cf5390a-8ad2-4842-8029-e242f88b0682"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:02.81385436Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1192,"":"downstairs","":"downstairs","session_id":"8cf5390a-8ad2-4842-8029-e242f88b0682"} {"msg":"[1]R flush_numbers[0..12]: [13, 14, 9, 13, 13, 8, 14, 16, 15, 16, 14, 11]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:02.81386512Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1192,"":"downstairs","":"downstairs","session_id":"8cf5390a-8ad2-4842-8029-e242f88b0682"} {"msg":"[1]R generation[0..12]: [10, 10, 8, 10, 10, 6, 10, 12, 11, 12, 10, 9]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:02.81387545Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1192,"":"downstairs","":"downstairs","session_id":"8cf5390a-8ad2-4842-8029-e242f88b0682"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:02.81388653Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1192,"":"downstairs","":"downstairs","session_id":"8cf5390a-8ad2-4842-8029-e242f88b0682"} {"msg":"[2]R flush_numbers[0..12]: [13, 14, 9, 13, 13, 8, 14, 16, 15, 16, 14, 11]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:02.81389714Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1192,"":"downstairs","":"downstairs","session_id":"8cf5390a-8ad2-4842-8029-e242f88b0682"} {"msg":"[2]R generation[0..12]: [10, 10, 8, 10, 10, 6, 10, 12, 11, 12, 10, 9]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:02.81390756Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1192,"":"downstairs","":"downstairs","session_id":"8cf5390a-8ad2-4842-8029-e242f88b0682"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:02.81391864Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1192,"":"downstairs","":"downstairs","session_id":"8cf5390a-8ad2-4842-8029-e242f88b0682"} {"msg":"Max found gen is 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:02.81392813Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1192,"":"downstairs","":"downstairs","session_id":"8cf5390a-8ad2-4842-8029-e242f88b0682"} {"msg":"Generation requested: 14 >= found:13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:02.81393788Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1192,"":"downstairs","":"downstairs","session_id":"8cf5390a-8ad2-4842-8029-e242f88b0682"} {"msg":"Next flush: 17","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:02.81394725Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1192,"":"downstairs","":"downstairs","session_id":"8cf5390a-8ad2-4842-8029-e242f88b0682"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:02.81395676Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1192,"":"downstairs","":"downstairs","session_id":"8cf5390a-8ad2-4842-8029-e242f88b0682"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:02.81396655Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1192,"session_id":"8cf5390a-8ad2-4842-8029-e242f88b0682"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:02.81397534Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1192,"client":"0","":"downstairs","session_id":"8cf5390a-8ad2-4842-8029-e242f88b0682"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:02.81398498Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1192,"client":"1","":"downstairs","session_id":"8cf5390a-8ad2-4842-8029-e242f88b0682"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:02.81399469Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1192,"client":"2","":"downstairs","session_id":"8cf5390a-8ad2-4842-8029-e242f88b0682"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:02.8140174Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1192,"session_id":"8cf5390a-8ad2-4842-8029-e242f88b0682"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:02.81402666Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1192,"session_id":"8cf5390a-8ad2-4842-8029-e242f88b0682"} {"msg":"65547ccf-f162-428f-864d-fbed82652e2c is now active with session: 8cf5390a-8ad2-4842-8029-e242f88b0682","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:02.81403679Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1192,"session_id":"8cf5390a-8ad2-4842-8029-e242f88b0682"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:02.814045751Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1192,"session_id":"8cf5390a-8ad2-4842-8029-e242f88b0682"} {"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:02.814055141Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1192} {"msg":"Activated sub_volume 65547ccf-f162-428f-864d-fbed82652e2c","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:02.814063241Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1192} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:02.814071441Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1192,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:02.814379371Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1192,"task":"crutest"} 01/30 Write block 359 len 40960 data: 5 5 5 3 2 2 2 2 2 2 02/30 Read block 107 len 36864 03/30 Write block 537 len 8192 data: 3 3 04/30 Read block 243 len 28672 05/30 Flush 06/30 Write block 154 len 32768 data: 3 3 3 2 2 2 2 2 07/30 Read block 276 len 28672 08/30 Write block 414 len 12288 data: 3 3 3 09/30 Write block 58 len 24576 data: 2 3 3 4 5 5 10/30 Write block 86 len 40960 data: 2 2 2 2 2 2 2 2 2 2 11/30 Read block 8 len 16384 12/30 Read block 298 len 8192 13/30 Write block 274 len 40960 data: 4 4 5 5 2 2 3 2 2 2 14/30 Read block 46 len 36864 15/30 Write block 147 len 24576 data: 3 3 4 4 4 4 16/30 Read block 441 len 24576 17/30 Write block 532 len 28672 data: 3 2 2 2 2 4 4 18/30 Flush 19/30 Flush 20/30 Write block 327 len 24576 data: 4 4 4 4 3 2 21/30 Flush 22/30 Flush 23/30 Write block 190 len 32768 data: 2 4 4 4 3 3 4 4 24/30 Write block 203 len 40960 data: 3 3 3 2 2 2 2 2 2 3 25/30 Read block 113 len 16384 26/30 Write block 253 len 12288 data: 2 3 3 27/30 Write block 569 len 12288 data: 5 5 4 28/30 Write block 90 len 36864 data: 3 3 3 3 3 3 2 2 2 29/30 Read block 258 len 40960 30/30 Read block 386 len 40960 ---------------------------------------------------------------- Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Read 9 Done Done Done false 1002 Acked Write 2 Done Sent Done false 1003 Acked Read 7 Done Sent Done false 1004 Acked Flush 0 Done Sent Done false 1005 Acked Write 8 Done Sent Done false 1006 Acked Read 7 Done Sent Done false 1007 Acked Write 3 Done Sent Done false 1008 Acked Write 6 Done Sent Done false 1009 Acked Write 10 Done Sent Done false 1010 Acked Read 4 Done Sent Done false 1011 Acked Read 2 Done Sent Done false 1012 Acked Write 10 Done Sent Done false 1013 Acked Read 9 Done Sent Done false 1014 Acked Write 6 Done Sent Done false 1015 Acked Read 6 Done Sent Done false 1016 Acked Write 7 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Write 6 Done Sent Done false 1020 Acked Flush 0 Done Sent Done false 1021 Acked Flush 0 Done Sent Done false 1022 Acked Write 8 Done Sent Done false 1023 Acked Write 10 Done Sent Done false 1024 Acked Read 4 Done Sent Done false 1025 Acked Write 3 Done Sent Done false 1026 Acked Write 3 Done Sent Done false 1027 Acked Write 9 Done Sent Done false 1028 Acked Read 10 Done Sent Done false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 28 1 29 Done 30 2 29 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) None Some(JobId(1021)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:02.823668945Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1192} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:02.823838435Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1192,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 13 13 13 F F F 1 020-039 10 10 10 14 14 14 F F F 2 040-059 14 8 14 18 9 18 F F F <--- 3 060-079 14 10 14 18 13 18 F F F <--- 4 080-099 14 10 14 18 13 18 T F T <--- 5 100-119 6 6 6 8 8 8 F F F 6 120-139 10 10 10 14 14 14 F F F 7 140-159 14 12 14 18 16 18 F F F <--- 8 160-179 14 11 14 18 15 18 F F F <--- 9 180-199 12 12 12 16 16 16 T F T <--- 10 200-219 10 10 10 14 14 14 T F T <--- 11 220-239 9 9 9 11 11 11 F F F 12 240-259 9 9 9 11 11 11 T F T <--- 13 260-279 14 9 14 18 11 18 F F F <--- 14 280-299 14 4 14 18 6 18 F F F <--- 15 300-319 5 5 5 7 7 7 F F F 16 320-339 14 12 14 20 16 20 F F F <--- 17 340-359 14 12 14 17 16 17 F T F <--- 18 360-379 14 12 14 17 16 17 F T F <--- 19 380-399 10 10 10 14 14 14 F F F 20 400-419 14 12 14 18 16 18 F F F <--- 21 420-439 12 12 12 16 16 16 F F F 22 440-459 2 2 2 4 4 4 F F F 23 460-479 10 10 10 12 12 12 F F F 24 480-499 12 12 12 16 16 16 F F F 25 500-519 12 12 12 16 16 16 F F F 26 520-539 14 12 14 18 16 18 F F F <--- 27 540-559 12 12 12 16 16 16 F F F 28 560-579 12 12 12 16 16 16 T F T <--- 29 580-599 1 1 1 1 1 1 F F F Max gen: 14, Max flush: 20 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 7 Verifying data now No change after verify Loop: 7 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 13 13 13 F F F 1 020-039 10 10 10 14 14 14 F F F 2 040-059 14 14 14 18 18 18 F F F 3 060-079 14 14 14 18 18 18 F F F 4 080-099 15 15 15 21 21 21 F F F 5 100-119 6 6 6 8 8 8 F F F 6 120-139 10 10 10 14 14 14 F F F 7 140-159 14 14 14 18 18 18 F F F 8 160-179 14 14 14 18 18 18 F F F 9 180-199 15 15 15 21 21 21 F F F 10 200-219 15 15 15 21 21 21 F F F 11 220-239 9 9 9 11 11 11 F F F 12 240-259 15 15 15 21 21 21 F F F 13 260-279 14 14 14 18 18 18 F F F 14 280-299 14 14 14 18 18 18 F F F 15 300-319 5 5 5 7 7 7 F F F 16 320-339 14 14 14 20 20 20 F F F 17 340-359 14 14 14 17 17 17 F F F 18 360-379 14 14 14 17 17 17 F F F 19 380-399 10 10 10 14 14 14 F F F 20 400-419 14 14 14 18 18 18 F F F 21 420-439 12 12 12 16 16 16 F F F 22 440-459 2 2 2 4 4 4 F F F 23 460-479 10 10 10 12 12 12 F F F 24 480-499 12 12 12 16 16 16 F F F 25 500-519 12 12 12 16 16 16 F F F 26 520-539 14 14 14 18 18 18 F F F 27 540-559 12 12 12 16 16 16 F F F 28 560-579 15 15 15 21 21 21 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 15, Max flush: 21 Begin loop 8 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:07.072715286Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1200,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:07.079945597Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1200,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:07.079999197Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1200,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:07.086862388Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1200,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:07.087133888Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1200} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:07.087160388Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1200} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:07.087169288Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1200} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:07.087181168Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1200} {"msg":"Crucible b5f4441c-9072-401f-aa77-f654ce4c755c has session id: 0e4e9c58-1bf6-4bef-b0ee-8888f4f9c849","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:07.087538589Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1200,"session_id":"0e4e9c58-1bf6-4bef-b0ee-8888f4f9c849"} {"msg":"Upstairs opts: Upstairs UUID: b5f4441c-9072-401f-aa77-f654ce4c755c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:07.087559589Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1200,"session_id":"0e4e9c58-1bf6-4bef-b0ee-8888f4f9c849"} {"msg":"Crucible stats registered with UUID: b5f4441c-9072-401f-aa77-f654ce4c755c","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:07.087569949Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1200,"session_id":"0e4e9c58-1bf6-4bef-b0ee-8888f4f9c849"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:07.087579099Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1200,"session_id":"0e4e9c58-1bf6-4bef-b0ee-8888f4f9c849"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:07.087721099Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1200,"":"io task","client":"1","":"downstairs","session_id":"0e4e9c58-1bf6-4bef-b0ee-8888f4f9c849"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:07.087738299Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1200,"":"io task","client":"0","":"downstairs","session_id":"0e4e9c58-1bf6-4bef-b0ee-8888f4f9c849"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:07.087748399Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1200,"":"io task","client":"2","":"downstairs","session_id":"0e4e9c58-1bf6-4bef-b0ee-8888f4f9c849"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:07.087891549Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1200,"task":"crutest"} {"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:07.087906919Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1200} {"msg":"b5f4441c-9072-401f-aa77-f654ce4c755c active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:07.087955249Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1200,"session_id":"0e4e9c58-1bf6-4bef-b0ee-8888f4f9c849"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:07.08796471Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1200,"client":"0","":"downstairs","session_id":"0e4e9c58-1bf6-4bef-b0ee-8888f4f9c849"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:07.08797481Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1200,"client":"1","":"downstairs","session_id":"0e4e9c58-1bf6-4bef-b0ee-8888f4f9c849"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:07.08798454Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1200,"client":"2","":"downstairs","session_id":"0e4e9c58-1bf6-4bef-b0ee-8888f4f9c849"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:07.08799425Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1200,"":"io task","client":"2","":"downstairs","session_id":"0e4e9c58-1bf6-4bef-b0ee-8888f4f9c849"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:07.0880242Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1200,"":"io task","client":"0","":"downstairs","session_id":"0e4e9c58-1bf6-4bef-b0ee-8888f4f9c849"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:07.08804646Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1200,"":"io task","client":"1","":"downstairs","session_id":"0e4e9c58-1bf6-4bef-b0ee-8888f4f9c849"} {"msg":"ds_connection connected from Ok(127.0.0.1:36056)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:07.08808289Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1200,"":"io task","client":"0","":"downstairs","session_id":"0e4e9c58-1bf6-4bef-b0ee-8888f4f9c849"} {"msg":"ds_connection connected from Ok(127.0.0.1:58586)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:07.08809421Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1200,"":"io task","client":"1","":"downstairs","session_id":"0e4e9c58-1bf6-4bef-b0ee-8888f4f9c849"} {"msg":"ds_connection connected from Ok(127.0.0.1:34468)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:07.08810468Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1200,"":"io task","client":"2","":"downstairs","session_id":"0e4e9c58-1bf6-4bef-b0ee-8888f4f9c849"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:07.088702301Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1200,"client":"1","":"downstairs","session_id":"0e4e9c58-1bf6-4bef-b0ee-8888f4f9c849"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:07.088751381Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1200,"client":"2","":"downstairs","session_id":"0e4e9c58-1bf6-4bef-b0ee-8888f4f9c849"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:07.088811041Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1200,"client":"0","":"downstairs","session_id":"0e4e9c58-1bf6-4bef-b0ee-8888f4f9c849"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:07.089035371Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1200,"session_id":"0e4e9c58-1bf6-4bef-b0ee-8888f4f9c849"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:07.089056921Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1200,"session_id":"0e4e9c58-1bf6-4bef-b0ee-8888f4f9c849"} {"msg":"[0]R flush_numbers[0..12]: [13, 14, 18, 18, 21, 8, 14, 18, 18, 21, 21, 11]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:07.089139341Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1200,"":"downstairs","":"downstairs","session_id":"0e4e9c58-1bf6-4bef-b0ee-8888f4f9c849"} {"msg":"[0]R generation[0..12]: [10, 10, 14, 14, 15, 6, 10, 14, 14, 15, 15, 9]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:07.089150871Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1200,"":"downstairs","":"downstairs","session_id":"0e4e9c58-1bf6-4bef-b0ee-8888f4f9c849"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:07.089162251Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1200,"":"downstairs","":"downstairs","session_id":"0e4e9c58-1bf6-4bef-b0ee-8888f4f9c849"} {"msg":"[1]R flush_numbers[0..12]: [13, 14, 18, 18, 21, 8, 14, 18, 18, 21, 21, 11]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:07.089172901Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1200,"":"downstairs","":"downstairs","session_id":"0e4e9c58-1bf6-4bef-b0ee-8888f4f9c849"} {"msg":"[1]R generation[0..12]: [10, 10, 14, 14, 15, 6, 10, 14, 14, 15, 15, 9]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:07.089183251Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1200,"":"downstairs","":"downstairs","session_id":"0e4e9c58-1bf6-4bef-b0ee-8888f4f9c849"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:07.089194471Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1200,"":"downstairs","":"downstairs","session_id":"0e4e9c58-1bf6-4bef-b0ee-8888f4f9c849"} {"msg":"[2]R flush_numbers[0..12]: [13, 14, 18, 18, 21, 8, 14, 18, 18, 21, 21, 11]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:07.089207051Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1200,"":"downstairs","":"downstairs","session_id":"0e4e9c58-1bf6-4bef-b0ee-8888f4f9c849"} {"msg":"[2]R generation[0..12]: [10, 10, 14, 14, 15, 6, 10, 14, 14, 15, 15, 9]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:07.089217501Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1200,"":"downstairs","":"downstairs","session_id":"0e4e9c58-1bf6-4bef-b0ee-8888f4f9c849"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:07.089228751Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1200,"":"downstairs","":"downstairs","session_id":"0e4e9c58-1bf6-4bef-b0ee-8888f4f9c849"} {"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:07.089238181Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1200,"":"downstairs","":"downstairs","session_id":"0e4e9c58-1bf6-4bef-b0ee-8888f4f9c849"} {"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:07.089248081Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1200,"":"downstairs","":"downstairs","session_id":"0e4e9c58-1bf6-4bef-b0ee-8888f4f9c849"} {"msg":"Next flush: 22","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:07.089257541Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1200,"":"downstairs","":"downstairs","session_id":"0e4e9c58-1bf6-4bef-b0ee-8888f4f9c849"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:07.089267061Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1200,"":"downstairs","":"downstairs","session_id":"0e4e9c58-1bf6-4bef-b0ee-8888f4f9c849"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:07.089277291Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1200,"session_id":"0e4e9c58-1bf6-4bef-b0ee-8888f4f9c849"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:07.089286301Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1200,"client":"0","":"downstairs","session_id":"0e4e9c58-1bf6-4bef-b0ee-8888f4f9c849"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:07.089296312Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1200,"client":"1","":"downstairs","session_id":"0e4e9c58-1bf6-4bef-b0ee-8888f4f9c849"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:07.089306242Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1200,"client":"2","":"downstairs","session_id":"0e4e9c58-1bf6-4bef-b0ee-8888f4f9c849"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:07.089316412Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1200,"session_id":"0e4e9c58-1bf6-4bef-b0ee-8888f4f9c849"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:07.089326052Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1200,"session_id":"0e4e9c58-1bf6-4bef-b0ee-8888f4f9c849"} {"msg":"b5f4441c-9072-401f-aa77-f654ce4c755c is now active with session: 0e4e9c58-1bf6-4bef-b0ee-8888f4f9c849","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:07.089336202Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1200,"session_id":"0e4e9c58-1bf6-4bef-b0ee-8888f4f9c849"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:07.089345062Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1200,"session_id":"0e4e9c58-1bf6-4bef-b0ee-8888f4f9c849"} {"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:07.089355522Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1200} {"msg":"Activated sub_volume b5f4441c-9072-401f-aa77-f654ce4c755c","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:07.089363632Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1200} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:07.089371802Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1200,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:07.089694862Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1200,"task":"crutest"} 01/30 Write block 149 len 32768 data: 5 5 5 5 4 4 4 4 02/30 Read block 53 len 28672 03/30 Read block 102 len 20480 04/30 Write block 51 len 32768 data: 4 3 3 2 2 2 2 3 05/30 Write block 289 len 8192 data: 2 2 06/30 Read block 547 len 4096 07/30 Read block 219 len 40960 08/30 Write block 272 len 36864 data: 4 4 5 5 6 6 3 3 4 09/30 Read block 358 len 40960 10/30 Write block 239 len 40960 data: 3 3 3 3 2 2 2 2 2 2 11/30 Read block 38 len 32768 12/30 Flush 13/30 Write block 301 len 40960 data: 4 4 4 3 3 3 3 2 2 2 14/30 Write block 255 len 8192 data: 4 2 15/30 Read block 122 len 24576 16/30 Write block 468 len 32768 data: 4 4 3 3 2 2 2 2 17/30 Read block 105 len 12288 18/30 Write block 154 len 32768 data: 5 5 5 3 3 3 3 3 19/30 Write block 575 len 32768 data: 3 3 4 4 3 2 2 2 20/30 Read block 230 len 32768 21/30 Write block 454 len 36864 data: 3 3 3 2 2 2 2 2 2 22/30 Flush 23/30 Write block 111 len 24576 data: 4 4 4 3 2 2 24/30 Read block 490 len 40960 25/30 Read block 370 len 12288 26/30 Read block 368 len 36864 27/30 Write block 520 len 40960 data: 2 2 2 3 3 3 3 4 4 4 28/30 Read block 121 len 32768 29/30 Read block 554 len 16384 30/30 Write block 215 len 8192 data: 2 3 ---------------------------------------------------------------- Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Sent Done Done false 1001 Acked Read 7 Sent Done Done false 1002 Acked Read 5 Sent Done Done false 1003 Acked Write 8 Sent Done Done false 1004 Acked Write 2 Sent Done Done false 1005 Acked Read 1 Sent Done Done false 1006 Acked Read 10 Sent Done Done false 1007 Acked Write 9 Sent Done Done false 1008 Acked Read 10 Sent Done Done false 1009 Acked Write 10 Sent Done Done false 1010 Acked Read 8 Sent Done Done false 1011 Acked Flush 0 Sent Done Done false 1012 Acked Write 10 Sent Done Done false 1013 Acked Write 2 Sent Done Done false 1014 Acked Read 6 Sent Done Done false 1015 Acked Write 8 Sent Done Done false 1016 Acked Read 3 Sent Done Done false 1017 Acked Write 8 Sent Done Done false 1018 Acked Write 8 Sent Done Done false 1019 Acked Read 8 Sent Done Done false 1020 Acked Write 9 Sent Done Done false 1021 Acked Flush 0 Sent Done Done false 1022 Acked Write 6 Sent Done Done false 1023 Acked Read 10 Sent Done Done false 1024 Acked Read 3 Sent Done Done false 1025 Acked Read 9 Sent Done Done false 1026 Acked Write 10 Sent Done Done false 1027 Acked Read 8 Sent Done Done false 1028 Acked Read 4 Sent Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 1 1 32 Done 0 29 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:07.100106438Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1200} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:07.100261028Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1200,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 13 13 13 F F F 1 020-039 10 10 10 14 14 14 F F F 2 040-059 14 16 16 18 22 22 F F F <--- 3 060-079 14 14 14 18 18 18 F F F 4 080-099 15 15 15 21 21 21 F F F 5 100-119 6 6 6 8 8 8 F T T <--- 6 120-139 10 10 10 14 14 14 F F F 7 140-159 14 16 16 18 23 23 F F F <--- 8 160-179 14 16 16 18 23 23 F F F <--- 9 180-199 15 15 15 21 21 21 F F F 10 200-219 15 15 15 21 21 21 F T T <--- 11 220-239 9 16 16 11 22 22 F F F <--- 12 240-259 15 16 16 21 23 23 F F F <--- 13 260-279 14 16 16 18 22 22 F F F <--- 14 280-299 14 16 16 18 22 22 F F F <--- 15 300-319 5 16 16 7 23 23 F F F <--- 16 320-339 14 14 14 20 20 20 F F F 17 340-359 14 14 14 17 17 17 F F F 18 360-379 14 14 14 17 17 17 F F F 19 380-399 10 10 10 14 14 14 F F F 20 400-419 14 14 14 18 18 18 F F F 21 420-439 12 12 12 16 16 16 F F F 22 440-459 2 16 16 4 23 23 F F F <--- 23 460-479 10 16 16 12 23 23 F F F <--- 24 480-499 12 12 12 16 16 16 F F F 25 500-519 12 12 12 16 16 16 F F F 26 520-539 14 14 14 18 18 18 F T T <--- 27 540-559 12 12 12 16 16 16 F F F 28 560-579 15 16 16 21 23 23 F F F <--- 29 580-599 1 16 16 1 23 23 F F F <--- Max gen: 16, Max flush: 23 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 8 Verifying data now No change after verify Loop: 8 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 13 13 13 F F F 1 020-039 10 10 10 14 14 14 F F F 2 040-059 16 16 16 22 22 22 F F F 3 060-079 14 14 14 18 18 18 F F F 4 080-099 15 15 15 21 21 21 F F F 5 100-119 17 17 17 24 24 24 F F F 6 120-139 10 10 10 14 14 14 F F F 7 140-159 16 16 16 23 23 23 F F F 8 160-179 16 16 16 23 23 23 F F F 9 180-199 15 15 15 21 21 21 F F F 10 200-219 17 17 17 24 24 24 F F F 11 220-239 16 16 16 22 22 22 F F F 12 240-259 16 16 16 23 23 23 F F F 13 260-279 16 16 16 22 22 22 F F F 14 280-299 16 16 16 22 22 22 F F F 15 300-319 16 16 16 23 23 23 F F F 16 320-339 14 14 14 20 20 20 F F F 17 340-359 14 14 14 17 17 17 F F F 18 360-379 14 14 14 17 17 17 F F F 19 380-399 10 10 10 14 14 14 F F F 20 400-419 14 14 14 18 18 18 F F F 21 420-439 12 12 12 16 16 16 F F F 22 440-459 16 16 16 23 23 23 F F F 23 460-479 16 16 16 23 23 23 F F F 24 480-499 12 12 12 16 16 16 F F F 25 500-519 12 12 12 16 16 16 F F F 26 520-539 17 17 17 24 24 24 F F F 27 540-559 12 12 12 16 16 16 F F F 28 560-579 16 16 16 23 23 23 F F F 29 580-599 16 16 16 23 23 23 F F F Max gen: 17, Max flush: 24 Begin loop 9 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:11.360450037Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1208,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:11.368823959Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1208,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:11.36914021Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1208} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:11.36916985Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1208} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:11.36917854Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1208} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:11.36919039Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1208} {"msg":"Crucible 43579779-6c4e-4ed8-8326-13ec5d0297c8 has session id: 25ae8be1-eb3d-4082-a5f4-1c74c4b1f33a","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:11.369564981Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1208,"session_id":"25ae8be1-eb3d-4082-a5f4-1c74c4b1f33a"} {"msg":"Upstairs opts: Upstairs UUID: 43579779-6c4e-4ed8-8326-13ec5d0297c8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:11.369593761Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1208,"session_id":"25ae8be1-eb3d-4082-a5f4-1c74c4b1f33a"} {"msg":"Crucible stats registered with UUID: 43579779-6c4e-4ed8-8326-13ec5d0297c8","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:11.369604931Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1208,"session_id":"25ae8be1-eb3d-4082-a5f4-1c74c4b1f33a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:11.369614401Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1208,"session_id":"25ae8be1-eb3d-4082-a5f4-1c74c4b1f33a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:11.369749211Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1208,"":"io task","client":"0","":"downstairs","session_id":"25ae8be1-eb3d-4082-a5f4-1c74c4b1f33a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:11.369782081Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1208,"":"io task","client":"1","":"downstairs","session_id":"25ae8be1-eb3d-4082-a5f4-1c74c4b1f33a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:11.369793071Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1208,"":"io task","client":"2","":"downstairs","session_id":"25ae8be1-eb3d-4082-a5f4-1c74c4b1f33a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:11.369930781Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1208,"task":"crutest"} {"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:11.369946701Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1208} {"msg":"43579779-6c4e-4ed8-8326-13ec5d0297c8 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:11.369962991Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1208,"session_id":"25ae8be1-eb3d-4082-a5f4-1c74c4b1f33a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:11.369971911Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1208,"client":"0","":"downstairs","session_id":"25ae8be1-eb3d-4082-a5f4-1c74c4b1f33a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:11.369981501Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1208,"client":"1","":"downstairs","session_id":"25ae8be1-eb3d-4082-a5f4-1c74c4b1f33a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:11.369991101Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1208,"client":"2","":"downstairs","session_id":"25ae8be1-eb3d-4082-a5f4-1c74c4b1f33a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:11.370000681Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1208,"":"io task","client":"2","":"downstairs","session_id":"25ae8be1-eb3d-4082-a5f4-1c74c4b1f33a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:11.370010641Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1208,"":"io task","client":"0","":"downstairs","session_id":"25ae8be1-eb3d-4082-a5f4-1c74c4b1f33a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:11.370058311Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1208,"":"io task","client":"1","":"downstairs","session_id":"25ae8be1-eb3d-4082-a5f4-1c74c4b1f33a"} {"msg":"ds_connection connected from Ok(127.0.0.1:50169)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:11.370078121Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1208,"":"io task","client":"2","":"downstairs","session_id":"25ae8be1-eb3d-4082-a5f4-1c74c4b1f33a"} {"msg":"ds_connection connected from Ok(127.0.0.1:47401)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:11.370221642Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1208,"":"io task","client":"1","":"downstairs","session_id":"25ae8be1-eb3d-4082-a5f4-1c74c4b1f33a"} {"msg":"ds_connection connected from Ok(127.0.0.1:41403)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:11.370242722Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1208,"":"io task","client":"0","":"downstairs","session_id":"25ae8be1-eb3d-4082-a5f4-1c74c4b1f33a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:11.370810532Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1208,"client":"2","":"downstairs","session_id":"25ae8be1-eb3d-4082-a5f4-1c74c4b1f33a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:11.370835982Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1208,"client":"1","":"downstairs","session_id":"25ae8be1-eb3d-4082-a5f4-1c74c4b1f33a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:11.370849813Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1208,"client":"0","":"downstairs","session_id":"25ae8be1-eb3d-4082-a5f4-1c74c4b1f33a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:11.371134513Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1208,"session_id":"25ae8be1-eb3d-4082-a5f4-1c74c4b1f33a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:11.371156873Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1208,"session_id":"25ae8be1-eb3d-4082-a5f4-1c74c4b1f33a"} {"msg":"[0]R flush_numbers[0..12]: [13, 14, 22, 18, 21, 24, 14, 23, 23, 21, 24, 22]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:11.371227773Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1208,"":"downstairs","":"downstairs","session_id":"25ae8be1-eb3d-4082-a5f4-1c74c4b1f33a"} {"msg":"[0]R generation[0..12]: [10, 10, 16, 14, 15, 17, 10, 16, 16, 15, 17, 16]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:11.371239453Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1208,"":"downstairs","":"downstairs","session_id":"25ae8be1-eb3d-4082-a5f4-1c74c4b1f33a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:11.371250823Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1208,"":"downstairs","":"downstairs","session_id":"25ae8be1-eb3d-4082-a5f4-1c74c4b1f33a"} {"msg":"[1]R flush_numbers[0..12]: [13, 14, 22, 18, 21, 24, 14, 23, 23, 21, 24, 22]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:11.371261403Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1208,"":"downstairs","":"downstairs","session_id":"25ae8be1-eb3d-4082-a5f4-1c74c4b1f33a"} {"msg":"[1]R generation[0..12]: [10, 10, 16, 14, 15, 17, 10, 16, 16, 15, 17, 16]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:11.371271833Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1208,"":"downstairs","":"downstairs","session_id":"25ae8be1-eb3d-4082-a5f4-1c74c4b1f33a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:11.371283013Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1208,"":"downstairs","":"downstairs","session_id":"25ae8be1-eb3d-4082-a5f4-1c74c4b1f33a"} {"msg":"[2]R flush_numbers[0..12]: [13, 14, 22, 18, 21, 24, 14, 23, 23, 21, 24, 22]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:11.371293583Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1208,"":"downstairs","":"downstairs","session_id":"25ae8be1-eb3d-4082-a5f4-1c74c4b1f33a"} {"msg":"[2]R generation[0..12]: [10, 10, 16, 14, 15, 17, 10, 16, 16, 15, 17, 16]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:11.371304063Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1208,"":"downstairs","":"downstairs","session_id":"25ae8be1-eb3d-4082-a5f4-1c74c4b1f33a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:11.371315133Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1208,"":"downstairs","":"downstairs","session_id":"25ae8be1-eb3d-4082-a5f4-1c74c4b1f33a"} {"msg":"Max found gen is 18","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:11.371324743Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1208,"":"downstairs","":"downstairs","session_id":"25ae8be1-eb3d-4082-a5f4-1c74c4b1f33a"} {"msg":"Generation requested: 18 >= found:18","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:11.371334583Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1208,"":"downstairs","":"downstairs","session_id":"25ae8be1-eb3d-4082-a5f4-1c74c4b1f33a"} {"msg":"Next flush: 25","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:11.371345983Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1208,"":"downstairs","":"downstairs","session_id":"25ae8be1-eb3d-4082-a5f4-1c74c4b1f33a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:11.371355403Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1208,"":"downstairs","":"downstairs","session_id":"25ae8be1-eb3d-4082-a5f4-1c74c4b1f33a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:11.371365283Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1208,"session_id":"25ae8be1-eb3d-4082-a5f4-1c74c4b1f33a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:11.371374113Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1208,"client":"0","":"downstairs","session_id":"25ae8be1-eb3d-4082-a5f4-1c74c4b1f33a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:11.371383853Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1208,"client":"1","":"downstairs","session_id":"25ae8be1-eb3d-4082-a5f4-1c74c4b1f33a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:11.371393493Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1208,"client":"2","":"downstairs","session_id":"25ae8be1-eb3d-4082-a5f4-1c74c4b1f33a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:11.371403413Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1208,"session_id":"25ae8be1-eb3d-4082-a5f4-1c74c4b1f33a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:11.371412723Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1208,"session_id":"25ae8be1-eb3d-4082-a5f4-1c74c4b1f33a"} {"msg":"43579779-6c4e-4ed8-8326-13ec5d0297c8 is now active with session: 25ae8be1-eb3d-4082-a5f4-1c74c4b1f33a","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:11.371422953Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1208,"session_id":"25ae8be1-eb3d-4082-a5f4-1c74c4b1f33a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:11.371431723Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1208,"session_id":"25ae8be1-eb3d-4082-a5f4-1c74c4b1f33a"} {"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:11.371440933Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1208} {"msg":"Activated sub_volume 43579779-6c4e-4ed8-8326-13ec5d0297c8","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:11.371449053Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1208} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:11.371457353Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1208,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:11.371780084Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1208,"task":"crutest"} 01/30 Write block 97 len 36864 data: 3 3 2 2 2 2 2 2 2 02/30 Write block 372 len 12288 data: 4 4 4 03/30 Flush 04/30 Write block 352 len 24576 data: 2 3 3 3 3 3 05/30 Write block 19 len 12288 data: 4 5 5 06/30 Flush 07/30 Write block 212 len 32768 data: 4 3 2 3 4 3 2 2 08/30 Write block 505 len 4096 data: 2 09/30 Write block 502 len 4096 data: 2 10/30 Read block 84 len 36864 11/30 Read block 210 len 28672 12/30 Write block 309 len 12288 data: 3 3 2 13/30 Read block 493 len 40960 14/30 Write block 299 len 40960 data: 3 4 5 5 5 4 4 4 4 3 15/30 Read block 82 len 8192 16/30 Read block 381 len 32768 17/30 Read block 495 len 40960 18/30 Write block 239 len 4096 data: 4 19/30 Read block 211 len 8192 20/30 Write block 137 len 32768 data: 3 3 3 3 3 2 2 3 21/30 Read block 568 len 40960 22/30 Flush 23/30 Flush 24/30 Write block 458 len 24576 data: 3 3 3 3 3 3 25/30 Read block 448 len 16384 26/30 Write block 79 len 36864 data: 3 3 2 2 2 2 2 3 3 27/30 Read block 179 len 28672 28/30 Write block 148 len 4096 data: 4 29/30 Write block 90 len 4096 data: 4 30/30 Write block 169 len 40960 data: 2 2 2 2 2 2 2 2 2 2 ---------------------------------------------------------------- Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 8 Done Done Done false 1007 Acked Write 1 Done Done Done false 1008 Acked Write 1 Done Done Done false 1009 Acked Read 9 Done Done Done false 1010 Acked Read 7 Done Done Done false 1011 Acked Write 3 Done Sent Done false 1012 Acked Read 10 Done Sent Done false 1013 Acked Write 10 Done Sent Done false 1014 Acked Read 2 Done Sent Done false 1015 Acked Read 8 Done Sent Done false 1016 Acked Read 10 Done Sent Done false 1017 Acked Write 1 Done Sent Done false 1018 Acked Read 2 Done Sent Done false 1019 Acked Write 8 Done Sent Done false 1020 Acked Read 10 Done Sent Done false 1021 Acked Flush 0 Done Sent Done false 1022 Acked Flush 0 Done Sent Done false 1023 Acked Write 6 Done Sent Done false 1024 Acked Read 4 Done Sent Done false 1025 Acked Write 9 Done Sent Done false 1026 Acked Read 7 Done Sent Done false 1027 Acked Write 1 Done Sent Done false 1028 Acked Write 1 Done Sent Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 19 1 21 Done 23 5 23 51 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1005)) Some(JobId(1022)) Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:11.380829358Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1208} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:11.381012858Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1208,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 18 18 18 26 26 26 F F F 1 020-039 18 18 18 26 26 26 F F F 2 040-059 16 16 16 22 22 22 F F F 3 060-079 14 14 14 18 18 18 T F T <--- 4 080-099 18 18 18 25 25 25 T F T <--- 5 100-119 18 18 18 25 25 25 F F F 6 120-139 18 10 18 27 14 27 F F F <--- 7 140-159 18 16 18 27 23 27 T F T <--- 8 160-179 16 16 16 23 23 23 T F T <--- 9 180-199 15 15 15 21 21 21 F F F 10 200-219 18 17 18 27 24 27 F T F <--- 11 220-239 18 16 18 27 22 27 F F F <--- 12 240-259 16 16 16 23 23 23 F F F 13 260-279 16 16 16 22 22 22 F F F 14 280-299 18 16 18 27 22 27 F F F <--- 15 300-319 18 16 18 27 23 27 F F F <--- 16 320-339 14 14 14 20 20 20 F F F 17 340-359 18 18 18 26 26 26 F F F 18 360-379 18 18 18 25 25 25 F F F 19 380-399 10 10 10 14 14 14 F F F 20 400-419 14 14 14 18 18 18 F F F 21 420-439 12 12 12 16 16 16 F F F 22 440-459 16 16 16 23 23 23 T F T <--- 23 460-479 16 16 16 23 23 23 T F T <--- 24 480-499 12 12 12 16 16 16 F F F 25 500-519 18 12 18 27 16 27 F T F <--- 26 520-539 17 17 17 24 24 24 F F F 27 540-559 12 12 12 16 16 16 F F F 28 560-579 16 16 16 23 23 23 F F F 29 580-599 16 16 16 23 23 23 F F F Max gen: 18, Max flush: 27 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 9 Verifying data now No change after verify Loop: 9 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 18 18 18 26 26 26 F F F 1 020-039 18 18 18 26 26 26 F F F 2 040-059 16 16 16 22 22 22 F F F 3 060-079 19 19 19 28 28 28 F F F 4 080-099 19 19 19 28 28 28 F F F 5 100-119 18 18 18 25 25 25 F F F 6 120-139 18 18 18 27 27 27 F F F 7 140-159 19 19 19 28 28 28 F F F 8 160-179 19 19 19 28 28 28 F F F 9 180-199 15 15 15 21 21 21 F F F 10 200-219 18 18 18 27 27 27 F F F 11 220-239 18 18 18 27 27 27 F F F 12 240-259 16 16 16 23 23 23 F F F 13 260-279 16 16 16 22 22 22 F F F 14 280-299 18 18 18 27 27 27 F F F 15 300-319 18 18 18 27 27 27 F F F 16 320-339 14 14 14 20 20 20 F F F 17 340-359 18 18 18 26 26 26 F F F 18 360-379 18 18 18 25 25 25 F F F 19 380-399 10 10 10 14 14 14 F F F 20 400-419 14 14 14 18 18 18 F F F 21 420-439 12 12 12 16 16 16 F F F 22 440-459 19 19 19 28 28 28 F F F 23 460-479 19 19 19 28 28 28 F F F 24 480-499 12 12 12 16 16 16 F F F 25 500-519 18 18 18 27 27 27 F F F 26 520-539 17 17 17 24 24 24 F F F 27 540-559 12 12 12 16 16 16 F F F 28 560-579 16 16 16 23 23 23 F F F 29 580-599 16 16 16 23 23 23 F F F Max gen: 19, Max flush: 28 Begin loop 10 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:15.647717936Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1216,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:15.655892949Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1216,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:15.656191289Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1216} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:15.656220259Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1216} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:15.656231149Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1216} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:15.656243019Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1216} {"msg":"Crucible eaa6b49c-6fec-4ebe-9c02-2dc99ba2b694 has session id: 6548436e-8b3f-43b4-a9d0-2558cab50327","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:15.65658592Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1216,"session_id":"6548436e-8b3f-43b4-a9d0-2558cab50327"} {"msg":"Upstairs opts: Upstairs UUID: eaa6b49c-6fec-4ebe-9c02-2dc99ba2b694, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:15.65663681Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1216,"session_id":"6548436e-8b3f-43b4-a9d0-2558cab50327"} {"msg":"Crucible stats registered with UUID: eaa6b49c-6fec-4ebe-9c02-2dc99ba2b694","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:15.65664748Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1216,"session_id":"6548436e-8b3f-43b4-a9d0-2558cab50327"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:15.65665646Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1216,"session_id":"6548436e-8b3f-43b4-a9d0-2558cab50327"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:15.65680144Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1216,"":"io task","client":"2","":"downstairs","session_id":"6548436e-8b3f-43b4-a9d0-2558cab50327"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:15.65681896Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1216,"":"io task","client":"0","":"downstairs","session_id":"6548436e-8b3f-43b4-a9d0-2558cab50327"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:15.65682901Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1216,"":"io task","client":"1","":"downstairs","session_id":"6548436e-8b3f-43b4-a9d0-2558cab50327"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:15.65697673Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1216,"task":"crutest"} {"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:15.6569921Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1216} {"msg":"eaa6b49c-6fec-4ebe-9c02-2dc99ba2b694 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:15.657043931Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1216,"session_id":"6548436e-8b3f-43b4-a9d0-2558cab50327"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:15.657070441Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1216,"client":"0","":"downstairs","session_id":"6548436e-8b3f-43b4-a9d0-2558cab50327"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:15.657081491Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1216,"client":"1","":"downstairs","session_id":"6548436e-8b3f-43b4-a9d0-2558cab50327"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:15.657091371Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1216,"client":"2","":"downstairs","session_id":"6548436e-8b3f-43b4-a9d0-2558cab50327"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:15.657103061Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1216,"":"io task","client":"2","":"downstairs","session_id":"6548436e-8b3f-43b4-a9d0-2558cab50327"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:15.657113391Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1216,"":"io task","client":"0","":"downstairs","session_id":"6548436e-8b3f-43b4-a9d0-2558cab50327"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:15.657123441Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1216,"":"io task","client":"1","":"downstairs","session_id":"6548436e-8b3f-43b4-a9d0-2558cab50327"} {"msg":"ds_connection connected from Ok(127.0.0.1:54889)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:15.657133791Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1216,"":"io task","client":"2","":"downstairs","session_id":"6548436e-8b3f-43b4-a9d0-2558cab50327"} {"msg":"ds_connection connected from Ok(127.0.0.1:36220)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:15.657144181Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1216,"":"io task","client":"0","":"downstairs","session_id":"6548436e-8b3f-43b4-a9d0-2558cab50327"} {"msg":"ds_connection connected from Ok(127.0.0.1:48229)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:15.657154691Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1216,"":"io task","client":"1","":"downstairs","session_id":"6548436e-8b3f-43b4-a9d0-2558cab50327"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:15.657811982Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1216,"client":"1","":"downstairs","session_id":"6548436e-8b3f-43b4-a9d0-2558cab50327"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:15.657855202Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1216,"client":"0","":"downstairs","session_id":"6548436e-8b3f-43b4-a9d0-2558cab50327"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:15.657905312Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1216,"client":"2","":"downstairs","session_id":"6548436e-8b3f-43b4-a9d0-2558cab50327"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:15.658114372Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1216,"session_id":"6548436e-8b3f-43b4-a9d0-2558cab50327"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:15.658131512Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1216,"session_id":"6548436e-8b3f-43b4-a9d0-2558cab50327"} {"msg":"[0]R flush_numbers[0..12]: [26, 26, 22, 28, 28, 25, 27, 28, 28, 21, 27, 27]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:15.658205132Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1216,"":"downstairs","":"downstairs","session_id":"6548436e-8b3f-43b4-a9d0-2558cab50327"} {"msg":"[0]R generation[0..12]: [18, 18, 16, 19, 19, 18, 18, 19, 19, 15, 18, 18]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:15.658216532Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1216,"":"downstairs","":"downstairs","session_id":"6548436e-8b3f-43b4-a9d0-2558cab50327"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:15.658227792Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1216,"":"downstairs","":"downstairs","session_id":"6548436e-8b3f-43b4-a9d0-2558cab50327"} {"msg":"[1]R flush_numbers[0..12]: [26, 26, 22, 28, 28, 25, 27, 28, 28, 21, 27, 27]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:15.658238472Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1216,"":"downstairs","":"downstairs","session_id":"6548436e-8b3f-43b4-a9d0-2558cab50327"} {"msg":"[1]R generation[0..12]: [18, 18, 16, 19, 19, 18, 18, 19, 19, 15, 18, 18]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:15.658251722Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1216,"":"downstairs","":"downstairs","session_id":"6548436e-8b3f-43b4-a9d0-2558cab50327"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:15.658262822Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1216,"":"downstairs","":"downstairs","session_id":"6548436e-8b3f-43b4-a9d0-2558cab50327"} {"msg":"[2]R flush_numbers[0..12]: [26, 26, 22, 28, 28, 25, 27, 28, 28, 21, 27, 27]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:15.658273392Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1216,"":"downstairs","":"downstairs","session_id":"6548436e-8b3f-43b4-a9d0-2558cab50327"} {"msg":"[2]R generation[0..12]: [18, 18, 16, 19, 19, 18, 18, 19, 19, 15, 18, 18]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:15.658283912Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1216,"":"downstairs","":"downstairs","session_id":"6548436e-8b3f-43b4-a9d0-2558cab50327"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:15.658294862Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1216,"":"downstairs","":"downstairs","session_id":"6548436e-8b3f-43b4-a9d0-2558cab50327"} {"msg":"Max found gen is 20","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:15.658304332Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1216,"":"downstairs","":"downstairs","session_id":"6548436e-8b3f-43b4-a9d0-2558cab50327"} {"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:15.658314112Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1216,"":"downstairs","":"downstairs","session_id":"6548436e-8b3f-43b4-a9d0-2558cab50327"} {"msg":"Next flush: 29","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:15.658323382Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1216,"":"downstairs","":"downstairs","session_id":"6548436e-8b3f-43b4-a9d0-2558cab50327"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:15.658332732Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1216,"":"downstairs","":"downstairs","session_id":"6548436e-8b3f-43b4-a9d0-2558cab50327"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:15.658342502Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1216,"session_id":"6548436e-8b3f-43b4-a9d0-2558cab50327"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:15.658351462Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1216,"client":"0","":"downstairs","session_id":"6548436e-8b3f-43b4-a9d0-2558cab50327"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:15.658361033Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1216,"client":"1","":"downstairs","session_id":"6548436e-8b3f-43b4-a9d0-2558cab50327"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:15.658370803Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1216,"client":"2","":"downstairs","session_id":"6548436e-8b3f-43b4-a9d0-2558cab50327"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:15.658380713Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1216,"session_id":"6548436e-8b3f-43b4-a9d0-2558cab50327"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:15.658389933Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1216,"session_id":"6548436e-8b3f-43b4-a9d0-2558cab50327"} {"msg":"eaa6b49c-6fec-4ebe-9c02-2dc99ba2b694 is now active with session: 6548436e-8b3f-43b4-a9d0-2558cab50327","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:15.658401293Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1216,"session_id":"6548436e-8b3f-43b4-a9d0-2558cab50327"} {"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:15.658410443Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1216} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:15.658418313Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1216,"session_id":"6548436e-8b3f-43b4-a9d0-2558cab50327"} {"msg":"Activated sub_volume eaa6b49c-6fec-4ebe-9c02-2dc99ba2b694","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:15.658427713Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1216} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:15.658435823Z","hostname":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 w-01KY5YGK9V792E6WY6KV93YHRV","pid":1216,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:15.658690593Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1216,"task":"crutest"} 01/30 Write block 362 len 24576 data: 4 3 3 3 3 3 02/30 Write block 285 len 12288 data: 2 2 2 03/30 Read block 535 len 20480 04/30 Write block 244 len 24576 data: 3 3 3 3 3 2 05/30 Write block 52 len 8192 data: 4 4 06/30 Write block 240 len 36864 data: 4 4 4 3 4 4 4 4 4 07/30 Read block 27 len 28672 08/30 Read block 247 len 40960 09/30 Read block 589 len 32768 10/30 Write block 263 len 28672 data: 2 2 2 3 3 4 5 11/30 Flush 12/30 Write block 555 len 28672 data: 2 2 2 3 2 2 2 13/30 Read block 498 len 4096 14/30 Read block 74 len 20480 15/30 Read block 52 len 36864 16/30 Read block 264 len 4096 17/30 Write block 19 len 28672 data: 5 6 6 5 5 5 4 18/30 Read block 33 len 8192 19/30 Write block 263 len 40960 data: 3 3 3 4 4 5 6 4 4 5 20/30 Write block 288 len 36864 data: 2 3 3 2 2 3 3 3 3 21/30 Write block 463 len 24576 data: 4 3 3 4 4 5 22/30 Write block 14 len 20480 data: 2 2 3 3 4 23/30 Read block 252 len 12288 24/30 Read block 268 len 24576 25/30 Write block 82 len 32768 data: 3 3 3 3 4 4 3 3 26/30 Write block 262 len 24576 data: 2 4 4 4 5 5 27/30 Read block 322 len 16384 28/30 Write block 208 len 16384 data: 3 3 3 3 29/30 Write block 181 len 32768 data: 3 2 2 2 2 2 2 2 30/30 Write block 548 len 8192 data: 2 2 ---------------------------------------------------------------- Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Sent false 1001 Acked Write 3 Done Done Sent false 1002 Acked Read 5 Done Done Sent false 1003 Acked Write 6 Done Done Sent false 1004 Acked Write 2 Done Done Sent false 1005 Acked Write 9 Done Done Sent false 1006 Acked Read 7 Done Done Sent false 1007 Acked Read 10 Done Done Sent false 1008 Acked Read 8 Done Done Sent false 1009 Acked Write 7 Done Done Sent false 1010 Acked Flush 0 Done Done Sent false 1011 Acked Write 7 Done Done Sent false 1012 Acked Read 1 Done Done Sent false 1013 Acked Read 5 Done Done Sent false 1014 Acked Read 9 Done Done Sent false 1015 Acked Read 1 Done Done Sent false 1016 Acked Write 7 Done Done Sent false 1017 Acked Read 2 Done Done Sent false 1018 Acked Write 10 Done Done Sent false 1019 Acked Write 9 Done Done Sent false 1020 Acked Write 6 Done Done Sent false 1021 Acked Write 5 Done Done Sent false 1022 Acked Read 3 Done Done Sent false 1023 Acked Read 6 Done Done Sent false 1024 Acked Write 8 Done Done Sent false 1025 Acked Write 6 Done Done Sent false 1026 Acked Read 4 Done Done Sent false 1027 Acked Write 4 Done Sent Sent false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 3 30 35 Done 28 27 0 55 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1010)) Some(JobId(1010)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:15.666927205Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1216} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:15.667110376Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1216,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 18 18 18 26 26 26 T T F <--- 1 020-039 18 18 18 26 26 26 T T F <--- 2 040-059 20 20 16 29 29 22 F F F <--- 3 060-079 19 19 19 28 28 28 F F F 4 080-099 19 19 19 28 28 28 T T F <--- 5 100-119 18 18 18 25 25 25 F F F 6 120-139 18 18 18 27 27 27 F F F 7 140-159 19 19 19 28 28 28 F F F 8 160-179 19 19 19 28 28 28 F F F 9 180-199 15 15 15 21 21 21 T T F <--- 10 200-219 18 18 18 27 27 27 T T F <--- 11 220-239 18 18 18 27 27 27 F F F 12 240-259 20 20 16 29 29 23 F F F <--- 13 260-279 20 20 16 29 29 22 T T F <--- 14 280-299 20 20 18 29 29 27 T T F <--- 15 300-319 18 18 18 27 27 27 F F F 16 320-339 14 14 14 20 20 20 F F F 17 340-359 18 18 18 26 26 26 F F F 18 360-379 20 20 18 29 29 25 F F F <--- 19 380-399 10 10 10 14 14 14 F F F 20 400-419 14 14 14 18 18 18 F F F 21 420-439 12 12 12 16 16 16 F F F 22 440-459 19 19 19 28 28 28 F F F 23 460-479 19 19 19 28 28 28 T T F <--- 24 480-499 12 12 12 16 16 16 F F F 25 500-519 18 18 18 27 27 27 F F F 26 520-539 17 17 17 24 24 24 F F F 27 540-559 12 12 12 16 16 16 T T F <--- 28 560-579 16 16 16 23 23 23 T T F <--- 29 580-599 16 16 16 23 23 23 F F F Max gen: 20, Max flush: 29 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 10 Verifying data now No change after verify Loop: 10 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 30 30 30 F F F 1 020-039 21 21 21 30 30 30 F F F 2 040-059 20 20 20 29 29 29 F F F 3 060-079 19 19 19 28 28 28 F F F 4 080-099 21 21 21 30 30 30 F F F 5 100-119 18 18 18 25 25 25 F F F 6 120-139 18 18 18 27 27 27 F F F 7 140-159 19 19 19 28 28 28 F F F 8 160-179 19 19 19 28 28 28 F F F 9 180-199 21 21 21 30 30 30 F F F 10 200-219 21 21 21 30 30 30 F F F 11 220-239 18 18 18 27 27 27 F F F 12 240-259 20 20 20 29 29 29 F F F 13 260-279 21 21 21 30 30 30 F F F 14 280-299 21 21 21 30 30 30 F F F 15 300-319 18 18 18 27 27 27 F F F 16 320-339 14 14 14 20 20 20 F F F 17 340-359 18 18 18 26 26 26 F F F 18 360-379 20 20 20 29 29 29 F F F 19 380-399 10 10 10 14 14 14 F F F 20 400-419 14 14 14 18 18 18 F F F 21 420-439 12 12 12 16 16 16 F F F 22 440-459 19 19 19 28 28 28 F F F 23 460-479 21 21 21 30 30 30 F F F 24 480-499 12 12 12 16 16 16 F F F 25 500-519 18 18 18 27 27 27 F F F 26 520-539 17 17 17 24 24 24 F F F 27 540-559 21 21 21 30 30 30 F F F 28 560-579 21 21 21 30 30 30 F F F 29 580-599 16 16 16 23 23 23 F F F Max gen: 21, Max flush: 30 Begin loop 11 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:19.91789828Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.925069851Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:19.925127971Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:19.932077571Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:19.932355822Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:19.932385642Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:19.932394482Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:19.932406122Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224} {"msg":"Crucible 07888c59-2b7c-4e64-8b3f-4b1db90307c8 has session id: 419433b9-bc3c-49e8-bd76-6d36d8443901","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:19.932778482Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"Upstairs opts: Upstairs UUID: 07888c59-2b7c-4e64-8b3f-4b1db90307c8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:19.932804292Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"Crucible stats registered with UUID: 07888c59-2b7c-4e64-8b3f-4b1db90307c8","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:19.932814612Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.932823882Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:19.932957783Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"":"io task","client":"2","":"downstairs","session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:19.932975553Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"":"io task","client":"0","":"downstairs","session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:19.932986573Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"":"io task","client":"1","":"downstairs","session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:19.933179653Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"task":"crutest"} {"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:19.933195123Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224} {"msg":"07888c59-2b7c-4e64-8b3f-4b1db90307c8 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:19.933213083Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:19.933222043Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"client":"0","":"downstairs","session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:19.933231683Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"client":"1","":"downstairs","session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:19.933241303Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"client":"2","":"downstairs","session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:19.933250973Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"":"io task","client":"2","":"downstairs","session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:19.933261123Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"":"io task","client":"0","":"downstairs","session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:19.933271113Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"":"io task","client":"1","":"downstairs","session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"ds_connection connected from Ok(127.0.0.1:38447)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:19.933347603Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"":"io task","client":"1","":"downstairs","session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"ds_connection connected from Ok(127.0.0.1:44527)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:19.933375153Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"":"io task","client":"2","":"downstairs","session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"ds_connection connected from Ok(127.0.0.1:65478)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:19.933386673Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"":"io task","client":"0","":"downstairs","session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:19.933989694Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"client":"2","":"downstairs","session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:19.934008614Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"client":"1","":"downstairs","session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:19.934019834Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"client":"0","":"downstairs","session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:19.934314715Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:19.934360125Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"[0]R flush_numbers[0..12]: [30, 30, 29, 28, 30, 25, 27, 28, 28, 30, 30, 27]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:19.934393735Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"":"downstairs","":"downstairs","session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"[0]R generation[0..12]: [21, 21, 20, 19, 21, 18, 18, 19, 19, 21, 21, 18]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:19.934404755Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"":"downstairs","":"downstairs","session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:19.934415915Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"":"downstairs","":"downstairs","session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"[1]R flush_numbers[0..12]: [30, 30, 29, 28, 30, 25, 27, 28, 28, 30, 30, 27]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:19.934426475Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"":"downstairs","":"downstairs","session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"[1]R generation[0..12]: [21, 21, 20, 19, 21, 18, 18, 19, 19, 21, 21, 18]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:19.934436905Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"":"downstairs","":"downstairs","session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:19.934448055Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"":"downstairs","":"downstairs","session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"[2]R flush_numbers[0..12]: [30, 30, 29, 28, 30, 25, 27, 28, 28, 30, 30, 27]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:19.934458775Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"":"downstairs","":"downstairs","session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"[2]R generation[0..12]: [21, 21, 20, 19, 21, 18, 18, 19, 19, 21, 21, 18]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:19.934471325Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"":"downstairs","":"downstairs","session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:19.934482535Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"":"downstairs","":"downstairs","session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"Max found gen is 22","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:19.934492215Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"":"downstairs","":"downstairs","session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"Generation requested: 22 >= found:22","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:19.934501985Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"":"downstairs","":"downstairs","session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"Next flush: 31","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:19.934511555Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"":"downstairs","":"downstairs","session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:19.934520985Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"":"downstairs","":"downstairs","session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:19.934530875Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:19.934539655Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"client":"0","":"downstairs","session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:19.934549385Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"client":"1","":"downstairs","session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:19.934559035Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"client":"2","":"downstairs","session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:19.934568985Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:19.934578135Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"07888c59-2b7c-4e64-8b3f-4b1db90307c8 is now active with session: 419433b9-bc3c-49e8-bd76-6d36d8443901","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:19.934588245Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:19.934597135Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:19.934606355Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224} {"msg":"Activated sub_volume 07888c59-2b7c-4e64-8b3f-4b1db90307c8","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:19.934614585Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:19.934623905Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:19.934930655Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"task":"crutest"} 01/30 Write block 248 len 16384 data: 5 3 2 2 02/30 Read block 39 len 12288 03/30 Read block 57 len 4096 04/30 Read block 562 len 16384 05/30 Flush 06/30 Write block 595 len 4096 data: 2 07/30 Read block 587 len 40960 08/30 Read block 94 len 36864 09/30 Read block 20 len 24576 10/30 Read block 113 len 16384 11/30 Write block 77 len 28672 data: 3 3 4 4 3 4 4 12/30 Write block 155 len 4096 data: 6 13/30 Write block 363 len 16384 data: 4 4 4 4 14/30 Read block 465 len 16384 15/30 Flush 16/30 Write block 174 len 20480 data: 3 3 3 3 3 17/30 Write block 19 len 28672 data: 6 7 7 6 6 6 5 18/30 Write block 293 len 16384 data: 4 4 4 4 19/30 Read block 480 len 4096 20/30 Write block 103 len 16384 data: 3 3 3 3 21/30 Write block 292 len 8192 data: 3 5 22/30 Write block 374 len 40960 data: 5 5 5 5 4 3 3 3 2 2 23/30 Read block 187 len 40960 24/30 Write block 505 len 16384 data: 3 2 2 2 25/30 Read block 404 len 32768 26/30 Read block 140 len 40960 27/30 Read block 66 len 12288 28/30 Write block 320 len 28672 data: 3 3 5 6 5 5 4 29/30 Write block 422 len 16384 data: 3 3 3 2 30/30 Write block 41 len 4096 data: 3 ---------------------------------------------------------------- Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Read 3 Done Done Done false 1002 Acked Read 1 Done Done Done false 1003 Acked Read 4 Sent Done Done false 1004 Acked Flush 0 Sent Done Done false 1005 Acked Write 1 Sent Done Done false 1006 Acked Read 10 Sent Done Done false 1007 Acked Read 9 Sent Done Done false 1008 Acked Read 6 Sent Done Done false 1009 Acked Read 4 Sent Done Done false 1010 Acked Write 7 Sent Done Done false 1011 Acked Write 1 Sent Done Done false 1012 Acked Write 4 Sent Done Done false 1013 Acked Read 4 Sent Done Done false 1014 Acked Flush 0 Sent Done Done false 1015 Acked Write 5 Sent Done Done false 1016 Acked Write 7 Sent Done Done false 1017 Acked Write 4 Sent Done Done false 1018 Acked Read 1 Sent Done Done false 1019 Acked Write 4 Sent Done Done false 1020 Acked Write 2 Sent Done Done false 1021 Acked Write 10 Sent Done Done false 1022 Acked Read 10 Sent Done Done false 1023 Acked Write 4 Sent Done Done false 1024 Acked Read 8 Sent Done Done false 1025 Acked Read 10 Sent Done Done false 1026 Acked Read 3 Sent Done Done false 1027 Acked Write 7 Sent Sent Sent false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 27 3 3 33 Done 3 27 27 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1014)) Some(JobId(1014)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.942492767Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:19.942649877Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"task":"crutest"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.942752927Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.942764977Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.942774237Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.942783617Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.942799937Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.942809157Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.942818337Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.942827437Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.942836577Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.942845667Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.942854847Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.942863897Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.942873037Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.942882228Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.942891368Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.942900628Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.942912518Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.942921768Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.942930938Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.942939948Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.942949078Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.942958208Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.942967418Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.942976678Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.942985888Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.942995038Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943004328Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943013598Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943022648Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943031898Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943040958Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943050298Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943059458Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943070108Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943079088Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943088268Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943097408Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943106488Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943115498Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943124528Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943133558Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943142638Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943151648Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943160888Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943169898Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943178878Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943187958Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943196958Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943206028Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943215258Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943225638Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943234608Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943243588Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943252578Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943261608Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943270618Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943279728Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943288788Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943297778Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943306748Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943315798Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943324838Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943333928Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943342888Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943352108Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943361068Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943370168Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943380268Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943389238Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943398238Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943407078Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943416068Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943424908Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943433908Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943442938Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943451838Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943460688Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943469598Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943478618Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943487588Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943496648Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943505708Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943514718Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943523858Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943534088Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943543138Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943552269Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943561269Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943570299Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943579299Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943588369Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943597359Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943606329Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943615319Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943624289Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943633409Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943642589Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943651659Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943660829Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943669879Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943678849Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943689059Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943698159Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943707259Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943716239Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943725219Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943734229Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943743189Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943752219Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943761209Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943770399Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943779379Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943788419Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943797449Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943806479Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943815509Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943824459Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943833489Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943843839Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943852929Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943861999Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943871009Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943880019Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:19.943888959Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1224,"session_id":"419433b9-bc3c-49e8-bd76-6d36d8443901"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 30 30 30 F T T <--- 1 020-039 21 21 21 30 30 30 F T T <--- 2 040-059 20 20 20 29 29 29 F T T <--- 3 060-079 19 22 22 28 32 32 F F F <--- 4 080-099 21 22 22 30 32 32 F F F <--- 5 100-119 18 18 18 25 25 25 F T T <--- 6 120-139 18 18 18 27 27 27 F F F 7 140-159 19 22 22 28 32 32 F F F <--- 8 160-179 19 19 19 28 28 28 F T T <--- 9 180-199 21 21 21 30 30 30 F F F 10 200-219 21 21 21 30 30 30 F F F 11 220-239 18 18 18 27 27 27 F F F 12 240-259 20 22 22 29 31 31 T F F <--- 13 260-279 21 21 21 30 30 30 F F F 14 280-299 21 21 21 30 30 30 F T T <--- 15 300-319 18 18 18 27 27 27 F F F 16 320-339 14 14 14 20 20 20 F T T <--- 17 340-359 18 18 18 26 26 26 F F F 18 360-379 20 22 22 29 32 32 F T T <--- 19 380-399 10 10 10 14 14 14 F T T <--- 20 400-419 14 14 14 18 18 18 F F F 21 420-439 12 12 12 16 16 16 F T T <--- 22 440-459 19 19 19 28 28 28 F F F 23 460-479 21 21 21 30 30 30 F F F 24 480-499 12 12 12 16 16 16 F F F 25 500-519 18 18 18 27 27 27 F T T <--- 26 520-539 17 17 17 24 24 24 F F F 27 540-559 21 21 21 30 30 30 F F F 28 560-579 21 21 21 30 30 30 F F F 29 580-599 16 22 22 23 32 32 F F F <--- Max gen: 22, Max flush: 32 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 11 Verifying data now No change after verify Loop: 11 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 23 23 23 33 33 33 F F F 1 020-039 23 23 23 33 33 33 F F F 2 040-059 23 23 23 33 33 33 F F F 3 060-079 22 22 22 32 32 32 F F F 4 080-099 22 22 22 32 32 32 F F F 5 100-119 23 23 23 33 33 33 F F F 6 120-139 18 18 18 27 27 27 F F F 7 140-159 22 22 22 32 32 32 F F F 8 160-179 23 23 23 33 33 33 F F F 9 180-199 21 21 21 30 30 30 F F F 10 200-219 21 21 21 30 30 30 F F F 11 220-239 18 18 18 27 27 27 F F F 12 240-259 22 22 22 31 31 31 F F F 13 260-279 21 21 21 30 30 30 F F F 14 280-299 23 23 23 33 33 33 F F F 15 300-319 18 18 18 27 27 27 F F F 16 320-339 23 23 23 33 33 33 F F F 17 340-359 18 18 18 26 26 26 F F F 18 360-379 23 23 23 33 33 33 F F F 19 380-399 23 23 23 33 33 33 F F F 20 400-419 14 14 14 18 18 18 F F F 21 420-439 23 23 23 33 33 33 F F F 22 440-459 19 19 19 28 28 28 F F F 23 460-479 21 21 21 30 30 30 F F F 24 480-499 12 12 12 16 16 16 F F F 25 500-519 23 23 23 33 33 33 F F F 26 520-539 17 17 17 24 24 24 F F F 27 540-559 21 21 21 30 30 30 F F F 28 560-579 21 21 21 30 30 30 F F F 29 580-599 22 22 22 32 32 32 F F F Max gen: 23, Max flush: 33 Begin loop 12 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:24.205104759Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1232,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:24.213503792Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1232,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:24.213814752Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1232} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:24.213841533Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1232} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:24.213850253Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1232} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:24.213861973Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1232} {"msg":"Crucible 2d311152-a897-4e4c-8180-497b66fcdb1e has session id: 9ad284c7-2804-4b86-b044-712b146651ed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:24.214241423Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1232,"session_id":"9ad284c7-2804-4b86-b044-712b146651ed"} {"msg":"Upstairs opts: Upstairs UUID: 2d311152-a897-4e4c-8180-497b66fcdb1e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:24.214263813Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1232,"session_id":"9ad284c7-2804-4b86-b044-712b146651ed"} {"msg":"Crucible stats registered with UUID: 2d311152-a897-4e4c-8180-497b66fcdb1e","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:24.214273683Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1232,"session_id":"9ad284c7-2804-4b86-b044-712b146651ed"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:24.214284993Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1232,"session_id":"9ad284c7-2804-4b86-b044-712b146651ed"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:24.214437103Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1232,"":"io task","client":"1","":"downstairs","session_id":"9ad284c7-2804-4b86-b044-712b146651ed"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:24.214469553Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1232,"":"io task","client":"0","":"downstairs","session_id":"9ad284c7-2804-4b86-b044-712b146651ed"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:24.214480603Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"9ad284c7-2804-4b86-b044-712b146651ed"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:24.214600224Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1232,"task":"crutest"} {"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:24.214615134Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1232} {"msg":"2d311152-a897-4e4c-8180-497b66fcdb1e active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:24.214653684Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1232,"session_id":"9ad284c7-2804-4b86-b044-712b146651ed"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:24.214663094Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1232,"client":"0","":"downstairs","session_id":"9ad284c7-2804-4b86-b044-712b146651ed"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:24.214672974Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1232,"client":"1","":"downstairs","session_id":"9ad284c7-2804-4b86-b044-712b146651ed"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:24.214682604Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1232,"client":"2","":"downstairs","session_id":"9ad284c7-2804-4b86-b044-712b146651ed"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:24.214692144Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"9ad284c7-2804-4b86-b044-712b146651ed"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:24.214702134Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1232,"":"io task","client":"0","":"downstairs","session_id":"9ad284c7-2804-4b86-b044-712b146651ed"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:24.214711984Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1232,"":"io task","client":"1","":"downstairs","session_id":"9ad284c7-2804-4b86-b044-712b146651ed"} {"msg":"ds_connection connected from Ok(127.0.0.1:49967)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:24.214768974Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1232,"":"io task","client":"0","":"downstairs","session_id":"9ad284c7-2804-4b86-b044-712b146651ed"} {"msg":"ds_connection connected from Ok(127.0.0.1:44501)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:24.214779864Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1232,"":"io task","client":"1","":"downstairs","session_id":"9ad284c7-2804-4b86-b044-712b146651ed"} {"msg":"ds_connection connected from Ok(127.0.0.1:47597)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:24.214790194Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"9ad284c7-2804-4b86-b044-712b146651ed"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:24.215447905Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1232,"client":"1","":"downstairs","session_id":"9ad284c7-2804-4b86-b044-712b146651ed"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:24.215472345Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1232,"client":"0","":"downstairs","session_id":"9ad284c7-2804-4b86-b044-712b146651ed"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:24.215495295Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1232,"client":"2","":"downstairs","session_id":"9ad284c7-2804-4b86-b044-712b146651ed"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:24.215770665Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1232,"session_id":"9ad284c7-2804-4b86-b044-712b146651ed"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:24.215780355Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1232,"session_id":"9ad284c7-2804-4b86-b044-712b146651ed"} {"msg":"[0]R flush_numbers[0..12]: [33, 33, 33, 32, 32, 33, 27, 32, 33, 30, 30, 27]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:24.215815995Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1232,"":"downstairs","":"downstairs","session_id":"9ad284c7-2804-4b86-b044-712b146651ed"} {"msg":"[0]R generation[0..12]: [23, 23, 23, 22, 22, 23, 18, 22, 23, 21, 21, 18]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:24.215827086Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1232,"":"downstairs","":"downstairs","session_id":"9ad284c7-2804-4b86-b044-712b146651ed"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:24.215838336Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1232,"":"downstairs","":"downstairs","session_id":"9ad284c7-2804-4b86-b044-712b146651ed"} {"msg":"[1]R flush_numbers[0..12]: [33, 33, 33, 32, 32, 33, 27, 32, 33, 30, 30, 27]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:24.215848786Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1232,"":"downstairs","":"downstairs","session_id":"9ad284c7-2804-4b86-b044-712b146651ed"} {"msg":"[1]R generation[0..12]: [23, 23, 23, 22, 22, 23, 18, 22, 23, 21, 21, 18]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:24.215859196Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1232,"":"downstairs","":"downstairs","session_id":"9ad284c7-2804-4b86-b044-712b146651ed"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:24.215870236Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1232,"":"downstairs","":"downstairs","session_id":"9ad284c7-2804-4b86-b044-712b146651ed"} {"msg":"[2]R flush_numbers[0..12]: [33, 33, 33, 32, 32, 33, 27, 32, 33, 30, 30, 27]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:24.215881106Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1232,"":"downstairs","":"downstairs","session_id":"9ad284c7-2804-4b86-b044-712b146651ed"} {"msg":"[2]R generation[0..12]: [23, 23, 23, 22, 22, 23, 18, 22, 23, 21, 21, 18]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:24.215891466Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1232,"":"downstairs","":"downstairs","session_id":"9ad284c7-2804-4b86-b044-712b146651ed"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:24.215902556Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1232,"":"downstairs","":"downstairs","session_id":"9ad284c7-2804-4b86-b044-712b146651ed"} {"msg":"Max found gen is 24","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:24.215914496Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1232,"":"downstairs","":"downstairs","session_id":"9ad284c7-2804-4b86-b044-712b146651ed"} {"msg":"Generation requested: 24 >= found:24","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:24.215924436Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1232,"":"downstairs","":"downstairs","session_id":"9ad284c7-2804-4b86-b044-712b146651ed"} {"msg":"Next flush: 34","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:24.215933836Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1232,"":"downstairs","":"downstairs","session_id":"9ad284c7-2804-4b86-b044-712b146651ed"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:24.215943356Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1232,"":"downstairs","":"downstairs","session_id":"9ad284c7-2804-4b86-b044-712b146651ed"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:24.215953526Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1232,"session_id":"9ad284c7-2804-4b86-b044-712b146651ed"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:24.215962666Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1232,"client":"0","":"downstairs","session_id":"9ad284c7-2804-4b86-b044-712b146651ed"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:24.215972886Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1232,"client":"1","":"downstairs","session_id":"9ad284c7-2804-4b86-b044-712b146651ed"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:24.215982806Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1232,"client":"2","":"downstairs","session_id":"9ad284c7-2804-4b86-b044-712b146651ed"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:24.215993006Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1232,"session_id":"9ad284c7-2804-4b86-b044-712b146651ed"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:24.216002486Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1232,"session_id":"9ad284c7-2804-4b86-b044-712b146651ed"} {"msg":"2d311152-a897-4e4c-8180-497b66fcdb1e is now active with session: 9ad284c7-2804-4b86-b044-712b146651ed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:24.216012726Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1232,"session_id":"9ad284c7-2804-4b86-b044-712b146651ed"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:24.216021586Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1232,"session_id":"9ad284c7-2804-4b86-b044-712b146651ed"} {"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:24.216030846Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1232} {"msg":"Activated sub_volume 2d311152-a897-4e4c-8180-497b66fcdb1e","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:24.216038956Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1232} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:24.216047316Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1232,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:24.216350596Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1232,"task":"crutest"} 01/30 Write block 492 len 20480 data: 4 3 3 2 2 02/30 Write block 139 len 16384 data: 4 4 4 3 03/30 Write block 185 len 24576 data: 3 3 3 3 2 3 04/30 Write block 255 len 36864 data: 5 3 2 2 2 2 2 3 5 05/30 Read block 43 len 4096 06/30 Read block 477 len 20480 07/30 Read block 285 len 4096 08/30 Read block 131 len 24576 09/30 Read block 371 len 28672 10/30 Read block 82 len 32768 11/30 Read block 124 len 20480 12/30 Write block 340 len 20480 data: 3 2 2 2 2 13/30 Read block 589 len 16384 14/30 Write block 538 len 8192 data: 5 4 15/30 Write block 535 len 28672 data: 3 3 5 6 5 3 3 16/30 Read block 116 len 20480 17/30 Write block 109 len 8192 data: 3 4 18/30 Read block 287 len 40960 19/30 Read block 582 len 4096 20/30 Read block 123 len 36864 21/30 Flush 22/30 Write block 489 len 28672 data: 4 3 3 5 4 4 3 23/30 Read block 419 len 40960 24/30 Read block 541 len 36864 25/30 Write block 385 len 24576 data: 2 2 2 4 4 4 26/30 Read block 151 len 20480 27/30 Read block 244 len 16384 28/30 Read block 256 len 8192 29/30 Read block 75 len 40960 30/30 Read block 76 len 24576 ---------------------------------------------------------------- Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Write 4 Done Done Done false 1002 Acked Write 6 Done Done Done false 1003 Acked Write 9 Done Done Done false 1004 Acked Read 1 Done Done Done false 1005 Acked Read 5 Done Done Done false 1006 Acked Read 1 Done Done Done false 1007 Acked Read 6 Done Sent Done false 1008 Acked Read 7 Done Sent Done false 1009 Acked Read 8 Done Sent Done false 1010 Acked Read 5 Done Sent Done false 1011 Acked Write 5 Done Sent Done false 1012 Acked Read 4 Done Sent Done false 1013 Acked Write 2 Done Sent Done false 1014 Acked Write 7 Done Sent Done false 1015 Acked Read 5 Done Sent Done false 1016 Acked Write 2 Done Sent Done false 1017 Acked Read 10 Done Sent Done false 1018 Acked Read 1 Done Sent Done false 1019 Acked Read 9 Done Sent Done false 1020 Acked Flush 0 Done Sent Done false 1021 Acked Write 7 Done Sent Done false 1022 Acked Read 10 Done Sent Done false 1023 Acked Read 9 Done Sent Done false 1024 Acked Write 6 Done Sent Done false 1025 Acked Read 5 Done Sent Done false 1026 Acked Read 4 Done Sent Done false 1027 Acked Read 2 Done Sent Done false 1028 Acked Read 10 Done Sent Done false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 23 1 24 Done 30 7 29 66 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) None Some(JobId(1020)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:24.22534776Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1232} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:24.22552178Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1232,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 23 23 23 33 33 33 F F F 1 020-039 23 23 23 33 33 33 F F F 2 040-059 23 23 23 33 33 33 F F F 3 060-079 22 22 22 32 32 32 F F F 4 080-099 22 22 22 32 32 32 F F F 5 100-119 24 23 24 34 33 34 F F F <--- 6 120-139 24 18 24 34 27 34 F T F <--- 7 140-159 24 22 24 34 32 34 F T F <--- 8 160-179 23 23 23 33 33 33 F F F 9 180-199 24 21 24 34 30 34 F T F <--- 10 200-219 21 21 21 30 30 30 F F F 11 220-239 18 18 18 27 27 27 F F F 12 240-259 24 22 24 34 31 34 F T F <--- 13 260-279 24 21 24 34 30 34 F T F <--- 14 280-299 23 23 23 33 33 33 F F F 15 300-319 18 18 18 27 27 27 F F F 16 320-339 23 23 23 33 33 33 F F F 17 340-359 24 18 24 34 26 34 F F F <--- 18 360-379 23 23 23 33 33 33 F F F 19 380-399 23 23 23 33 33 33 T F T <--- 20 400-419 14 14 14 18 18 18 F F F 21 420-439 23 23 23 33 33 33 F F F 22 440-459 19 19 19 28 28 28 F F F 23 460-479 21 21 21 30 30 30 F F F 24 480-499 24 12 24 34 16 34 T T T <--- 25 500-519 23 23 23 33 33 33 F F F 26 520-539 24 17 24 34 24 34 F F F <--- 27 540-559 24 21 24 34 30 34 F F F <--- 28 560-579 21 21 21 30 30 30 F F F 29 580-599 22 22 22 32 32 32 F F F Max gen: 24, Max flush: 34 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 12 Verifying data now No change after verify Loop: 12 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 23 23 23 33 33 33 F F F 1 020-039 23 23 23 33 33 33 F F F 2 040-059 23 23 23 33 33 33 F F F 3 060-079 22 22 22 32 32 32 F F F 4 080-099 22 22 22 32 32 32 F F F 5 100-119 24 24 24 34 34 34 F F F 6 120-139 24 24 24 34 34 34 F F F 7 140-159 24 24 24 34 34 34 F F F 8 160-179 23 23 23 33 33 33 F F F 9 180-199 24 24 24 34 34 34 F F F 10 200-219 21 21 21 30 30 30 F F F 11 220-239 18 18 18 27 27 27 F F F 12 240-259 24 24 24 34 34 34 F F F 13 260-279 24 24 24 34 34 34 F F F 14 280-299 23 23 23 33 33 33 F F F 15 300-319 18 18 18 27 27 27 F F F 16 320-339 23 23 23 33 33 33 F F F 17 340-359 24 24 24 34 34 34 F F F 18 360-379 23 23 23 33 33 33 F F F 19 380-399 25 25 25 35 35 35 F F F 20 400-419 14 14 14 18 18 18 F F F 21 420-439 23 23 23 33 33 33 F F F 22 440-459 19 19 19 28 28 28 F F F 23 460-479 21 21 21 30 30 30 F F F 24 480-499 25 25 25 35 35 35 F F F 25 500-519 23 23 23 33 33 33 F F F 26 520-539 24 24 24 34 34 34 F F F 27 540-559 24 24 24 34 34 34 F F F 28 560-579 21 21 21 30 30 30 F F F 29 580-599 22 22 22 32 32 32 F F F Max gen: 25, Max flush: 35 Begin loop 13 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:28.471275347Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1240,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:28.478408048Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1240,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:28.478462048Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1240,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:28.485312088Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1240,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:28.485577719Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1240} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:28.485604649Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1240} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:28.485612929Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1240} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:28.485624769Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1240} {"msg":"Crucible 56f67e42-2eb7-4815-80df-621f2b707f69 has session id: 8efe79d5-0d09-4f66-b6db-717e398f4d2c","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:28.485986209Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1240,"session_id":"8efe79d5-0d09-4f66-b6db-717e398f4d2c"} {"msg":"Upstairs opts: Upstairs UUID: 56f67e42-2eb7-4815-80df-621f2b707f69, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:28.486024349Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1240,"session_id":"8efe79d5-0d09-4f66-b6db-717e398f4d2c"} {"msg":"Crucible stats registered with UUID: 56f67e42-2eb7-4815-80df-621f2b707f69","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:28.486035479Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1240,"session_id":"8efe79d5-0d09-4f66-b6db-717e398f4d2c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:28.486044779Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1240,"session_id":"8efe79d5-0d09-4f66-b6db-717e398f4d2c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:28.48617475Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1240,"":"io task","client":"1","":"downstairs","session_id":"8efe79d5-0d09-4f66-b6db-717e398f4d2c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:28.48620677Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1240,"":"io task","client":"0","":"downstairs","session_id":"8efe79d5-0d09-4f66-b6db-717e398f4d2c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:28.48621783Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1240,"":"io task","client":"2","":"downstairs","session_id":"8efe79d5-0d09-4f66-b6db-717e398f4d2c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:28.4863549Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1240,"task":"crutest"} {"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:28.48636956Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1240} {"msg":"56f67e42-2eb7-4815-80df-621f2b707f69 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:28.48640391Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1240,"session_id":"8efe79d5-0d09-4f66-b6db-717e398f4d2c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:28.48641353Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1240,"client":"0","":"downstairs","session_id":"8efe79d5-0d09-4f66-b6db-717e398f4d2c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:28.48642347Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1240,"client":"1","":"downstairs","session_id":"8efe79d5-0d09-4f66-b6db-717e398f4d2c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:28.4864332Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1240,"client":"2","":"downstairs","session_id":"8efe79d5-0d09-4f66-b6db-717e398f4d2c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:28.48644286Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1240,"":"io task","client":"2","":"downstairs","session_id":"8efe79d5-0d09-4f66-b6db-717e398f4d2c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:28.48646404Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1240,"":"io task","client":"0","":"downstairs","session_id":"8efe79d5-0d09-4f66-b6db-717e398f4d2c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:28.4865006Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1240,"":"io task","client":"1","":"downstairs","session_id":"8efe79d5-0d09-4f66-b6db-717e398f4d2c"} {"msg":"ds_connection connected from Ok(127.0.0.1:56306)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:28.48651881Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1240,"":"io task","client":"2","":"downstairs","session_id":"8efe79d5-0d09-4f66-b6db-717e398f4d2c"} {"msg":"ds_connection connected from Ok(127.0.0.1:55863)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:28.48654778Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1240,"":"io task","client":"1","":"downstairs","session_id":"8efe79d5-0d09-4f66-b6db-717e398f4d2c"} {"msg":"ds_connection connected from Ok(127.0.0.1:37890)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:28.48657039Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1240,"":"io task","client":"0","":"downstairs","session_id":"8efe79d5-0d09-4f66-b6db-717e398f4d2c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:28.487225751Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1240,"client":"0","":"downstairs","session_id":"8efe79d5-0d09-4f66-b6db-717e398f4d2c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:28.487245111Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1240,"client":"2","":"downstairs","session_id":"8efe79d5-0d09-4f66-b6db-717e398f4d2c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:28.487256231Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1240,"client":"1","":"downstairs","session_id":"8efe79d5-0d09-4f66-b6db-717e398f4d2c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:28.487545762Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1240,"session_id":"8efe79d5-0d09-4f66-b6db-717e398f4d2c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:28.487570042Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1240,"session_id":"8efe79d5-0d09-4f66-b6db-717e398f4d2c"} {"msg":"[0]R flush_numbers[0..12]: [33, 33, 33, 32, 32, 34, 34, 34, 33, 34, 30, 27]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:28.487638062Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1240,"":"downstairs","":"downstairs","session_id":"8efe79d5-0d09-4f66-b6db-717e398f4d2c"} {"msg":"[0]R generation[0..12]: [23, 23, 23, 22, 22, 24, 24, 24, 23, 24, 21, 18]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:28.487649632Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1240,"":"downstairs","":"downstairs","session_id":"8efe79d5-0d09-4f66-b6db-717e398f4d2c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:28.487660932Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1240,"":"downstairs","":"downstairs","session_id":"8efe79d5-0d09-4f66-b6db-717e398f4d2c"} {"msg":"[1]R flush_numbers[0..12]: [33, 33, 33, 32, 32, 34, 34, 34, 33, 34, 30, 27]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:28.487671372Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1240,"":"downstairs","":"downstairs","session_id":"8efe79d5-0d09-4f66-b6db-717e398f4d2c"} {"msg":"[1]R generation[0..12]: [23, 23, 23, 22, 22, 24, 24, 24, 23, 24, 21, 18]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:28.487681872Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1240,"":"downstairs","":"downstairs","session_id":"8efe79d5-0d09-4f66-b6db-717e398f4d2c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:28.487692992Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1240,"":"downstairs","":"downstairs","session_id":"8efe79d5-0d09-4f66-b6db-717e398f4d2c"} {"msg":"[2]R flush_numbers[0..12]: [33, 33, 33, 32, 32, 34, 34, 34, 33, 34, 30, 27]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:28.487703532Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1240,"":"downstairs","":"downstairs","session_id":"8efe79d5-0d09-4f66-b6db-717e398f4d2c"} {"msg":"[2]R generation[0..12]: [23, 23, 23, 22, 22, 24, 24, 24, 23, 24, 21, 18]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:28.487713902Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1240,"":"downstairs","":"downstairs","session_id":"8efe79d5-0d09-4f66-b6db-717e398f4d2c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:28.487724952Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1240,"":"downstairs","":"downstairs","session_id":"8efe79d5-0d09-4f66-b6db-717e398f4d2c"} {"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:28.487734572Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1240,"":"downstairs","":"downstairs","session_id":"8efe79d5-0d09-4f66-b6db-717e398f4d2c"} {"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:28.487744532Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1240,"":"downstairs","":"downstairs","session_id":"8efe79d5-0d09-4f66-b6db-717e398f4d2c"} {"msg":"Next flush: 36","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:28.487754002Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1240,"":"downstairs","":"downstairs","session_id":"8efe79d5-0d09-4f66-b6db-717e398f4d2c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:28.487763372Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1240,"":"downstairs","":"downstairs","session_id":"8efe79d5-0d09-4f66-b6db-717e398f4d2c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:28.487773212Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1240,"session_id":"8efe79d5-0d09-4f66-b6db-717e398f4d2c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:28.487783762Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1240,"client":"0","":"downstairs","session_id":"8efe79d5-0d09-4f66-b6db-717e398f4d2c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:28.487793642Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1240,"client":"1","":"downstairs","session_id":"8efe79d5-0d09-4f66-b6db-717e398f4d2c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:28.487803292Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1240,"client":"2","":"downstairs","session_id":"8efe79d5-0d09-4f66-b6db-717e398f4d2c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:28.487813282Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1240,"session_id":"8efe79d5-0d09-4f66-b6db-717e398f4d2c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:28.487822532Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1240,"session_id":"8efe79d5-0d09-4f66-b6db-717e398f4d2c"} {"msg":"56f67e42-2eb7-4815-80df-621f2b707f69 is now active with session: 8efe79d5-0d09-4f66-b6db-717e398f4d2c","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:28.487832582Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1240,"session_id":"8efe79d5-0d09-4f66-b6db-717e398f4d2c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:28.487841372Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1240,"session_id":"8efe79d5-0d09-4f66-b6db-717e398f4d2c"} {"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:28.487850662Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1240} {"msg":"Activated sub_volume 56f67e42-2eb7-4815-80df-621f2b707f69","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:28.487858732Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1240} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:28.487866962Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1240,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:28.488165533Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1240,"task":"crutest"} 01/30 Write block 368 len 36864 data: 3 2 3 3 5 5 6 6 6 02/30 Write block 389 len 4096 data: 5 03/30 Read block 534 len 24576 04/30 Write block 422 len 32768 data: 4 4 4 3 2 2 2 2 05/30 Write block 160 len 8192 data: 4 4 06/30 Read block 327 len 16384 07/30 Read block 521 len 16384 08/30 Write block 408 len 4096 data: 2 09/30 Write block 539 len 24576 data: 6 4 4 3 2 2 10/30 Read block 485 len 32768 11/30 Write block 334 len 16384 data: 3 4 3 4 12/30 Write block 297 len 12288 data: 3 3 4 13/30 Read block 101 len 28672 14/30 Write block 325 len 16384 data: 6 5 5 5 15/30 Read block 408 len 16384 16/30 Read block 117 len 16384 17/30 Flush 18/30 Write block 456 len 16384 data: 4 3 4 4 19/30 Write block 121 len 12288 data: 3 3 3 20/30 Read block 588 len 36864 21/30 Write block 339 len 12288 data: 3 4 3 22/30 Read block 530 len 32768 23/30 Write block 105 len 32768 data: 4 4 3 3 4 5 5 5 24/30 Read block 23 len 16384 25/30 Read block 5 len 32768 26/30 Write block 36 len 16384 data: 3 2 2 3 27/30 Read block 442 len 24576 28/30 Write block 488 len 12288 data: 3 5 4 29/30 Read block 21 len 28672 30/30 Write block 154 len 8192 data: 6 7 ---------------------------------------------------------------- Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Write 1 Done Done Done false 1002 Acked Read 6 Sent Done Done false 1003 Acked Write 8 Sent Done Done false 1004 Acked Write 2 Sent Done Done false 1005 Acked Read 4 Sent Done Done false 1006 Acked Read 4 Sent Done Done false 1007 Acked Write 1 Sent Done Done false 1008 Acked Write 6 Sent Done Done false 1009 Acked Read 8 Sent Done Done false 1010 Acked Write 4 Sent Done Done false 1011 Acked Write 3 Sent Done Done false 1012 Acked Read 7 Sent Done Done false 1013 Acked Write 4 Sent Done Done false 1014 Acked Read 4 Sent Done Done false 1015 Acked Read 4 Sent Done Done false 1016 Acked Flush 0 Sent Done Done false 1017 Acked Write 4 Sent Done Done false 1018 Acked Write 3 Sent Done Done false 1019 Acked Read 9 Sent Done Done false 1020 Acked Write 3 Sent Done Done false 1021 Acked Read 8 Sent Done Done false 1022 Acked Write 8 Sent Done Done false 1023 Acked Read 4 Sent Done Done false 1024 Acked Read 8 Sent Done Done false 1025 Acked Write 4 Sent Done Done false 1026 Acked Read 6 Sent Done Done false 1027 Acked Write 3 Sent Done Done false 1028 Acked Read 7 Sent Done Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 1 2 31 Done 2 29 28 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:28.496722606Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1240} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:28.496913716Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1240,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 23 23 23 33 33 33 F F F 1 020-039 23 23 23 33 33 33 F T T <--- 2 040-059 23 23 23 33 33 33 F F F 3 060-079 22 22 22 32 32 32 F F F 4 080-099 22 22 22 32 32 32 F F F 5 100-119 24 24 24 34 34 34 F T T <--- 6 120-139 24 24 24 34 34 34 F T T <--- 7 140-159 24 24 24 34 34 34 F T T <--- 8 160-179 23 26 26 33 36 36 F F F <--- 9 180-199 24 24 24 34 34 34 F F F 10 200-219 21 21 21 30 30 30 F F F 11 220-239 18 18 18 27 27 27 F F F 12 240-259 24 24 24 34 34 34 F F F 13 260-279 24 24 24 34 34 34 F F F 14 280-299 23 26 26 33 36 36 F F F <--- 15 300-319 18 18 18 27 27 27 F F F 16 320-339 23 26 26 33 36 36 F T T <--- 17 340-359 24 24 24 34 34 34 F T T <--- 18 360-379 23 26 26 33 36 36 T F F <--- 19 380-399 25 26 26 35 36 36 T F F <--- 20 400-419 14 26 26 18 36 36 F F F <--- 21 420-439 23 26 26 33 36 36 F F F <--- 22 440-459 19 19 19 28 28 28 F T T <--- 23 460-479 21 21 21 30 30 30 F F F 24 480-499 25 25 25 35 35 35 F T T <--- 25 500-519 23 23 23 33 33 33 F F F 26 520-539 24 26 26 34 36 36 F F F <--- 27 540-559 24 26 26 34 36 36 F F F <--- 28 560-579 21 21 21 30 30 30 F F F 29 580-599 22 22 22 32 32 32 F F F Max gen: 26, Max flush: 36 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 13 Verifying data now No change after verify Loop: 13 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 23 23 23 33 33 33 F F F 1 020-039 27 27 27 37 37 37 F F F 2 040-059 23 23 23 33 33 33 F F F 3 060-079 22 22 22 32 32 32 F F F 4 080-099 22 22 22 32 32 32 F F F 5 100-119 27 27 27 37 37 37 F F F 6 120-139 27 27 27 37 37 37 F F F 7 140-159 27 27 27 37 37 37 F F F 8 160-179 26 26 26 36 36 36 F F F 9 180-199 24 24 24 34 34 34 F F F 10 200-219 21 21 21 30 30 30 F F F 11 220-239 18 18 18 27 27 27 F F F 12 240-259 24 24 24 34 34 34 F F F 13 260-279 24 24 24 34 34 34 F F F 14 280-299 26 26 26 36 36 36 F F F 15 300-319 18 18 18 27 27 27 F F F 16 320-339 27 27 27 37 37 37 F F F 17 340-359 27 27 27 37 37 37 F F F 18 360-379 26 26 26 36 36 36 F F F 19 380-399 26 26 26 36 36 36 F F F 20 400-419 26 26 26 36 36 36 F F F 21 420-439 26 26 26 36 36 36 F F F 22 440-459 27 27 27 37 37 37 F F F 23 460-479 21 21 21 30 30 30 F F F 24 480-499 27 27 27 37 37 37 F F F 25 500-519 23 23 23 33 33 33 F F F 26 520-539 26 26 26 36 36 36 F F F 27 540-559 26 26 26 36 36 36 F F F 28 560-579 21 21 21 30 30 30 F F F 29 580-599 22 22 22 32 32 32 F F F Max gen: 27, Max flush: 37 Begin loop 14 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:32.763088284Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1248,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:32.771443896Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1248,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:32.771731067Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1248} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:32.771758737Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1248} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:32.771767237Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1248} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:32.771781707Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1248} {"msg":"Crucible 367000bf-d35a-46e9-81e0-a76990a73aa2 has session id: 4790290d-58e5-440a-88cd-295878bc8564","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:32.772177587Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1248,"session_id":"4790290d-58e5-440a-88cd-295878bc8564"} {"msg":"Upstairs opts: Upstairs UUID: 367000bf-d35a-46e9-81e0-a76990a73aa2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:32.772224857Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1248,"session_id":"4790290d-58e5-440a-88cd-295878bc8564"} {"msg":"Crucible stats registered with UUID: 367000bf-d35a-46e9-81e0-a76990a73aa2","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:32.772235747Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1248,"session_id":"4790290d-58e5-440a-88cd-295878bc8564"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:32.772244837Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1248,"session_id":"4790290d-58e5-440a-88cd-295878bc8564"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:32.772381668Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1248,"":"io task","client":"0","":"downstairs","session_id":"4790290d-58e5-440a-88cd-295878bc8564"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:32.772413968Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1248,"":"io task","client":"1","":"downstairs","session_id":"4790290d-58e5-440a-88cd-295878bc8564"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:32.772425148Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1248,"":"io task","client":"2","":"downstairs","session_id":"4790290d-58e5-440a-88cd-295878bc8564"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:32.772548108Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1248,"task":"crutest"} {"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:32.772562538Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1248} {"msg":"367000bf-d35a-46e9-81e0-a76990a73aa2 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:32.772597008Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1248,"session_id":"4790290d-58e5-440a-88cd-295878bc8564"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:32.772606518Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1248,"client":"0","":"downstairs","session_id":"4790290d-58e5-440a-88cd-295878bc8564"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:32.772616338Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1248,"client":"1","":"downstairs","session_id":"4790290d-58e5-440a-88cd-295878bc8564"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:32.772626028Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1248,"client":"2","":"downstairs","session_id":"4790290d-58e5-440a-88cd-295878bc8564"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:32.772635698Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1248,"":"io task","client":"2","":"downstairs","session_id":"4790290d-58e5-440a-88cd-295878bc8564"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:32.772647588Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1248,"":"io task","client":"0","":"downstairs","session_id":"4790290d-58e5-440a-88cd-295878bc8564"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:32.772657668Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1248,"":"io task","client":"1","":"downstairs","session_id":"4790290d-58e5-440a-88cd-295878bc8564"} {"msg":"ds_connection connected from Ok(127.0.0.1:51705)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:32.772724728Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1248,"":"io task","client":"0","":"downstairs","session_id":"4790290d-58e5-440a-88cd-295878bc8564"} {"msg":"ds_connection connected from Ok(127.0.0.1:51957)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:32.772748668Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1248,"":"io task","client":"1","":"downstairs","session_id":"4790290d-58e5-440a-88cd-295878bc8564"} {"msg":"ds_connection connected from Ok(127.0.0.1:42091)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:32.772760018Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1248,"":"io task","client":"2","":"downstairs","session_id":"4790290d-58e5-440a-88cd-295878bc8564"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:32.773399259Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1248,"client":"0","":"downstairs","session_id":"4790290d-58e5-440a-88cd-295878bc8564"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:32.773426169Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1248,"client":"1","":"downstairs","session_id":"4790290d-58e5-440a-88cd-295878bc8564"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:32.773445209Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1248,"client":"2","":"downstairs","session_id":"4790290d-58e5-440a-88cd-295878bc8564"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:32.77369898Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1248,"session_id":"4790290d-58e5-440a-88cd-295878bc8564"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:32.7737343Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1248,"session_id":"4790290d-58e5-440a-88cd-295878bc8564"} {"msg":"[0]R flush_numbers[0..12]: [33, 37, 33, 32, 32, 37, 37, 37, 36, 34, 30, 27]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:32.77378171Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1248,"":"downstairs","":"downstairs","session_id":"4790290d-58e5-440a-88cd-295878bc8564"} {"msg":"[0]R generation[0..12]: [23, 27, 23, 22, 22, 27, 27, 27, 26, 24, 21, 18]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:32.77379287Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1248,"":"downstairs","":"downstairs","session_id":"4790290d-58e5-440a-88cd-295878bc8564"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:32.77380405Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1248,"":"downstairs","":"downstairs","session_id":"4790290d-58e5-440a-88cd-295878bc8564"} {"msg":"[1]R flush_numbers[0..12]: [33, 37, 33, 32, 32, 37, 37, 37, 36, 34, 30, 27]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:32.77381466Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1248,"":"downstairs","":"downstairs","session_id":"4790290d-58e5-440a-88cd-295878bc8564"} {"msg":"[1]R generation[0..12]: [23, 27, 23, 22, 22, 27, 27, 27, 26, 24, 21, 18]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:32.77382517Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1248,"":"downstairs","":"downstairs","session_id":"4790290d-58e5-440a-88cd-295878bc8564"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:32.77383847Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1248,"":"downstairs","":"downstairs","session_id":"4790290d-58e5-440a-88cd-295878bc8564"} {"msg":"[2]R flush_numbers[0..12]: [33, 37, 33, 32, 32, 37, 37, 37, 36, 34, 30, 27]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:32.7738491Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1248,"":"downstairs","":"downstairs","session_id":"4790290d-58e5-440a-88cd-295878bc8564"} {"msg":"[2]R generation[0..12]: [23, 27, 23, 22, 22, 27, 27, 27, 26, 24, 21, 18]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:32.77385959Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1248,"":"downstairs","":"downstairs","session_id":"4790290d-58e5-440a-88cd-295878bc8564"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:32.77387058Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1248,"":"downstairs","":"downstairs","session_id":"4790290d-58e5-440a-88cd-295878bc8564"} {"msg":"Max found gen is 28","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:32.77388006Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1248,"":"downstairs","":"downstairs","session_id":"4790290d-58e5-440a-88cd-295878bc8564"} {"msg":"Generation requested: 28 >= found:28","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:32.77388994Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1248,"":"downstairs","":"downstairs","session_id":"4790290d-58e5-440a-88cd-295878bc8564"} {"msg":"Next flush: 38","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:32.77389932Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1248,"":"downstairs","":"downstairs","session_id":"4790290d-58e5-440a-88cd-295878bc8564"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:32.7739088Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1248,"":"downstairs","":"downstairs","session_id":"4790290d-58e5-440a-88cd-295878bc8564"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:32.77391853Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1248,"session_id":"4790290d-58e5-440a-88cd-295878bc8564"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:32.77392747Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1248,"client":"0","":"downstairs","session_id":"4790290d-58e5-440a-88cd-295878bc8564"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:32.77393724Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1248,"client":"1","":"downstairs","session_id":"4790290d-58e5-440a-88cd-295878bc8564"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:32.77394705Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1248,"client":"2","":"downstairs","session_id":"4790290d-58e5-440a-88cd-295878bc8564"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:32.77395693Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1248,"session_id":"4790290d-58e5-440a-88cd-295878bc8564"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:32.77396617Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1248,"session_id":"4790290d-58e5-440a-88cd-295878bc8564"} {"msg":"367000bf-d35a-46e9-81e0-a76990a73aa2 is now active with session: 4790290d-58e5-440a-88cd-295878bc8564","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:32.77397628Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1248,"session_id":"4790290d-58e5-440a-88cd-295878bc8564"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:32.77398639Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1248,"session_id":"4790290d-58e5-440a-88cd-295878bc8564"} {"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:32.77399563Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1248} {"msg":"Activated sub_volume 367000bf-d35a-46e9-81e0-a76990a73aa2","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:32.77400382Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1248} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:32.77401212Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1248,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:32.774298781Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1248,"task":"crutest"} 01/30 Write block 559 len 8192 data: 3 3 02/30 Read block 564 len 24576 03/30 Write block 320 len 40960 data: 4 4 6 7 6 7 6 6 6 5 04/30 Read block 241 len 40960 05/30 Read block 113 len 36864 06/30 Read block 334 len 24576 07/30 Write block 287 len 16384 data: 3 3 4 4 08/30 Flush 09/30 Write block 50 len 12288 data: 4 5 5 10/30 Read block 187 len 36864 11/30 Read block 481 len 36864 12/30 Write block 216 len 4096 data: 5 13/30 Read block 69 len 16384 14/30 Write block 421 len 4096 data: 3 15/30 Write block 517 len 36864 data: 2 2 2 3 3 3 4 4 4 16/30 Read block 254 len 40960 17/30 Read block 456 len 20480 18/30 Read block 169 len 4096 19/30 Read block 261 len 16384 20/30 Read block 90 len 8192 21/30 Write block 138 len 8192 data: 4 5 22/30 Write block 323 len 40960 data: 8 7 8 7 7 7 6 5 4 3 23/30 Write block 414 len 4096 data: 4 24/30 Write block 377 len 36864 data: 6 5 4 4 4 3 3 2 3 25/30 Read block 5 len 32768 26/30 Read block 439 len 12288 27/30 Read block 526 len 12288 28/30 Read block 243 len 36864 29/30 Write block 85 len 8192 data: 4 5 30/30 Write block 187 len 28672 data: 4 4 3 4 5 5 5 ---------------------------------------------------------------- Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:22 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1008 Acked Write 3 Done Done Done false 1009 Acked Read 9 Done Done Done false 1010 Acked Read 9 Done Done Done false 1011 Acked Write 1 Done Done Done false 1012 Acked Read 4 Done Done Done false 1013 Acked Write 1 Done Sent Done false 1014 Acked Write 9 Done Sent Done false 1015 Acked Read 10 Done Sent Done false 1016 Acked Read 5 Done Sent Done false 1017 Acked Read 1 Done Sent Done false 1018 Acked Read 4 Done Sent Done false 1019 Acked Read 2 Done Sent Done false 1020 Acked Write 2 Done Sent Done false 1021 Acked Write 10 Done Sent Done false 1022 Acked Write 1 Done Sent Done false 1023 Acked Write 9 Done Sent Done false 1024 Acked Read 8 Done Sent Done false 1025 Acked Read 3 Done Sent Done false 1026 Acked Read 3 Done Sent Done false 1027 Acked Read 9 Done Sent Done false 1028 Acked Write 2 Done Sent Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 17 1 19 Done 21 5 21 47 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1007)) Some(JobId(1007)) Some(JobId(1007)) Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:32.783095684Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1248} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:32.783286784Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1248,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 23 23 23 33 33 33 F F F 1 020-039 27 27 27 37 37 37 F F F 2 040-059 23 23 23 33 33 33 T T T 3 060-079 22 22 22 32 32 32 F F F 4 080-099 22 22 22 32 32 32 T F T <--- 5 100-119 27 27 27 37 37 37 F F F 6 120-139 27 27 27 37 37 37 T F T <--- 7 140-159 27 27 27 37 37 37 F F F 8 160-179 26 26 26 36 36 36 F F F 9 180-199 24 24 24 34 34 34 T F T <--- 10 200-219 21 21 21 30 30 30 T T T 11 220-239 18 18 18 27 27 27 F F F 12 240-259 24 24 24 34 34 34 F F F 13 260-279 24 24 24 34 34 34 F F F 14 280-299 28 28 28 38 38 38 F F F 15 300-319 18 18 18 27 27 27 F F F 16 320-339 28 28 28 38 38 38 T F T <--- 17 340-359 27 27 27 37 37 37 F F F 18 360-379 26 26 26 36 36 36 T F T <--- 19 380-399 26 26 26 36 36 36 T F T <--- 20 400-419 26 26 26 36 36 36 T F T <--- 21 420-439 26 26 26 36 36 36 T F T <--- 22 440-459 27 27 27 37 37 37 F F F 23 460-479 21 21 21 30 30 30 F F F 24 480-499 27 27 27 37 37 37 F F F 25 500-519 23 23 23 33 33 33 T F T <--- 26 520-539 26 26 26 36 36 36 T F T <--- 27 540-559 28 28 28 38 38 38 F F F 28 560-579 28 28 28 38 38 38 F F F 29 580-599 22 22 22 32 32 32 F F F Max gen: 28, Max flush: 38 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 14 Verifying data now No change after verify Loop: 14 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 23 23 23 33 33 33 F F F 1 020-039 27 27 27 37 37 37 F F F 2 040-059 29 29 29 39 39 39 F F F 3 060-079 22 22 22 32 32 32 F F F 4 080-099 29 29 29 39 39 39 F F F 5 100-119 27 27 27 37 37 37 F F F 6 120-139 29 29 29 39 39 39 F F F 7 140-159 27 27 27 37 37 37 F F F 8 160-179 26 26 26 36 36 36 F F F 9 180-199 29 29 29 39 39 39 F F F 10 200-219 29 29 29 39 39 39 F F F 11 220-239 18 18 18 27 27 27 F F F 12 240-259 24 24 24 34 34 34 F F F 13 260-279 24 24 24 34 34 34 F F F 14 280-299 28 28 28 38 38 38 F F F 15 300-319 18 18 18 27 27 27 F F F 16 320-339 29 29 29 39 39 39 F F F 17 340-359 27 27 27 37 37 37 F F F 18 360-379 29 29 29 39 39 39 F F F 19 380-399 29 29 29 39 39 39 F F F 20 400-419 29 29 29 39 39 39 F F F 21 420-439 29 29 29 39 39 39 F F F 22 440-459 27 27 27 37 37 37 F F F 23 460-479 21 21 21 30 30 30 F F F 24 480-499 27 27 27 37 37 37 F F F 25 500-519 29 29 29 39 39 39 F F F 26 520-539 29 29 29 39 39 39 F F F 27 540-559 28 28 28 38 38 38 F F F 28 560-579 28 28 28 38 38 38 F F F 29 580-599 22 22 22 32 32 32 F F F Max gen: 29, Max flush: 39 Begin loop 15 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:37.034563439Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1256,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:37.0417692Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1256,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:37.0418187Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1256,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:37.04867561Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1256,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:37.048944091Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1256} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:37.048979401Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1256} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:37.048989061Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1256} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:37.049011491Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1256} {"msg":"Crucible 88da8fa5-d9a6-4d3f-9b29-a88cf9cc9202 has session id: 2838b3de-5c14-495b-8d18-cc51f08d7c04","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:37.049304981Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1256,"session_id":"2838b3de-5c14-495b-8d18-cc51f08d7c04"} {"msg":"Upstairs opts: Upstairs UUID: 88da8fa5-d9a6-4d3f-9b29-a88cf9cc9202, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:37.049328881Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1256,"session_id":"2838b3de-5c14-495b-8d18-cc51f08d7c04"} {"msg":"Crucible stats registered with UUID: 88da8fa5-d9a6-4d3f-9b29-a88cf9cc9202","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:37.049339311Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1256,"session_id":"2838b3de-5c14-495b-8d18-cc51f08d7c04"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:37.049348611Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1256,"session_id":"2838b3de-5c14-495b-8d18-cc51f08d7c04"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:37.049493531Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1256,"":"io task","client":"1","":"downstairs","session_id":"2838b3de-5c14-495b-8d18-cc51f08d7c04"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:37.049525531Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1256,"":"io task","client":"0","":"downstairs","session_id":"2838b3de-5c14-495b-8d18-cc51f08d7c04"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:37.049536641Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1256,"":"io task","client":"2","":"downstairs","session_id":"2838b3de-5c14-495b-8d18-cc51f08d7c04"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:37.049658842Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1256,"task":"crutest"} {"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:37.049677212Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1256} {"msg":"88da8fa5-d9a6-4d3f-9b29-a88cf9cc9202 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:37.049708212Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1256,"session_id":"2838b3de-5c14-495b-8d18-cc51f08d7c04"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:37.049717722Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1256,"client":"0","":"downstairs","session_id":"2838b3de-5c14-495b-8d18-cc51f08d7c04"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:37.049727362Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1256,"client":"1","":"downstairs","session_id":"2838b3de-5c14-495b-8d18-cc51f08d7c04"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:37.049737132Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1256,"client":"2","":"downstairs","session_id":"2838b3de-5c14-495b-8d18-cc51f08d7c04"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:37.049746732Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1256,"":"io task","client":"2","":"downstairs","session_id":"2838b3de-5c14-495b-8d18-cc51f08d7c04"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:37.049756752Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1256,"":"io task","client":"0","":"downstairs","session_id":"2838b3de-5c14-495b-8d18-cc51f08d7c04"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:37.049795722Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1256,"":"io task","client":"1","":"downstairs","session_id":"2838b3de-5c14-495b-8d18-cc51f08d7c04"} {"msg":"ds_connection connected from Ok(127.0.0.1:48433)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:37.049812002Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1256,"":"io task","client":"1","":"downstairs","session_id":"2838b3de-5c14-495b-8d18-cc51f08d7c04"} {"msg":"ds_connection connected from Ok(127.0.0.1:57489)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:37.049941892Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1256,"":"io task","client":"2","":"downstairs","session_id":"2838b3de-5c14-495b-8d18-cc51f08d7c04"} {"msg":"ds_connection connected from Ok(127.0.0.1:51507)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:37.049966292Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1256,"":"io task","client":"0","":"downstairs","session_id":"2838b3de-5c14-495b-8d18-cc51f08d7c04"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:37.050403353Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1256,"client":"1","":"downstairs","session_id":"2838b3de-5c14-495b-8d18-cc51f08d7c04"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:37.050475563Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1256,"client":"2","":"downstairs","session_id":"2838b3de-5c14-495b-8d18-cc51f08d7c04"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:37.050507203Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1256,"client":"0","":"downstairs","session_id":"2838b3de-5c14-495b-8d18-cc51f08d7c04"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:37.050704823Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1256,"session_id":"2838b3de-5c14-495b-8d18-cc51f08d7c04"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:37.050721823Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1256,"session_id":"2838b3de-5c14-495b-8d18-cc51f08d7c04"} {"msg":"[0]R flush_numbers[0..12]: [33, 37, 39, 32, 39, 37, 39, 37, 36, 39, 39, 27]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:37.050783923Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1256,"":"downstairs","":"downstairs","session_id":"2838b3de-5c14-495b-8d18-cc51f08d7c04"} {"msg":"[0]R generation[0..12]: [23, 27, 29, 22, 29, 27, 29, 27, 26, 29, 29, 18]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:37.050795313Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1256,"":"downstairs","":"downstairs","session_id":"2838b3de-5c14-495b-8d18-cc51f08d7c04"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:37.050806473Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1256,"":"downstairs","":"downstairs","session_id":"2838b3de-5c14-495b-8d18-cc51f08d7c04"} {"msg":"[1]R flush_numbers[0..12]: [33, 37, 39, 32, 39, 37, 39, 37, 36, 39, 39, 27]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:37.050817293Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1256,"":"downstairs","":"downstairs","session_id":"2838b3de-5c14-495b-8d18-cc51f08d7c04"} {"msg":"[1]R generation[0..12]: [23, 27, 29, 22, 29, 27, 29, 27, 26, 29, 29, 18]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:37.050827723Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1256,"":"downstairs","":"downstairs","session_id":"2838b3de-5c14-495b-8d18-cc51f08d7c04"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:37.050838923Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1256,"":"downstairs","":"downstairs","session_id":"2838b3de-5c14-495b-8d18-cc51f08d7c04"} {"msg":"[2]R flush_numbers[0..12]: [33, 37, 39, 32, 39, 37, 39, 37, 36, 39, 39, 27]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:37.050859323Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1256,"":"downstairs","":"downstairs","session_id":"2838b3de-5c14-495b-8d18-cc51f08d7c04"} {"msg":"[2]R generation[0..12]: [23, 27, 29, 22, 29, 27, 29, 27, 26, 29, 29, 18]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:37.050869864Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1256,"":"downstairs","":"downstairs","session_id":"2838b3de-5c14-495b-8d18-cc51f08d7c04"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:37.050880874Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1256,"":"downstairs","":"downstairs","session_id":"2838b3de-5c14-495b-8d18-cc51f08d7c04"} {"msg":"Max found gen is 30","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:37.050890424Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1256,"":"downstairs","":"downstairs","session_id":"2838b3de-5c14-495b-8d18-cc51f08d7c04"} {"msg":"Generation requested: 30 >= found:30","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:37.050905954Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1256,"":"downstairs","":"downstairs","session_id":"2838b3de-5c14-495b-8d18-cc51f08d7c04"} {"msg":"Next flush: 40","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:37.050915464Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1256,"":"downstairs","":"downstairs","session_id":"2838b3de-5c14-495b-8d18-cc51f08d7c04"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:37.050924974Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1256,"":"downstairs","":"downstairs","session_id":"2838b3de-5c14-495b-8d18-cc51f08d7c04"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:37.050934864Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1256,"session_id":"2838b3de-5c14-495b-8d18-cc51f08d7c04"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:37.050943694Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1256,"client":"0","":"downstairs","session_id":"2838b3de-5c14-495b-8d18-cc51f08d7c04"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:37.050953414Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1256,"client":"1","":"downstairs","session_id":"2838b3de-5c14-495b-8d18-cc51f08d7c04"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:37.050966674Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1256,"client":"2","":"downstairs","session_id":"2838b3de-5c14-495b-8d18-cc51f08d7c04"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:37.050976534Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1256,"session_id":"2838b3de-5c14-495b-8d18-cc51f08d7c04"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:37.050985764Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1256,"session_id":"2838b3de-5c14-495b-8d18-cc51f08d7c04"} {"msg":"88da8fa5-d9a6-4d3f-9b29-a88cf9cc9202 is now active with session: 2838b3de-5c14-495b-8d18-cc51f08d7c04","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:37.050995804Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1256,"session_id":"2838b3de-5c14-495b-8d18-cc51f08d7c04"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:37.051004654Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1256,"session_id":"2838b3de-5c14-495b-8d18-cc51f08d7c04"} {"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:37.051013844Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1256} {"msg":"Activated sub_volume 88da8fa5-d9a6-4d3f-9b29-a88cf9cc9202","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:37.051022084Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1256} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:37.051036284Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1256,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:37.051330254Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1256,"task":"crutest"} 01/30 Flush 02/30 Write block 185 len 16384 data: 4 4 5 5 03/30 Read block 493 len 12288 04/30 Read block 247 len 16384 05/30 Read block 131 len 32768 06/30 Write block 519 len 40960 data: 3 4 4 4 5 5 5 4 5 5 07/30 Write block 447 len 12288 data: 2 3 3 08/30 Read block 283 len 16384 09/30 Read block 330 len 4096 10/30 Write block 218 len 8192 data: 3 3 11/30 Read block 77 len 40960 12/30 Read block 6 len 32768 13/30 Write block 316 len 16384 data: 2 2 2 2 14/30 Read block 52 len 36864 15/30 Flush 16/30 Write block 146 len 16384 data: 3 4 5 6 17/30 Read block 80 len 32768 18/30 Read block 330 len 24576 19/30 Write block 583 len 32768 data: 2 2 2 2 2 2 2 2 20/30 Write block 455 len 20480 data: 4 5 4 5 5 21/30 Read block 365 len 8192 22/30 Write block 236 len 32768 data: 3 3 3 5 5 5 5 4 23/30 Read block 541 len 16384 24/30 Write block 209 len 40960 data: 4 4 4 5 4 3 4 6 4 4 25/30 Read block 255 len 8192 26/30 Read block 508 len 8192 27/30 Read block 489 len 36864 28/30 Write block 95 len 20480 data: 4 3 4 4 3 29/30 Write block 528 len 16384 data: 6 5 3 3 30/30 Write block 187 len 12288 data: 6 6 4 ---------------------------------------------------------------- Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 4 Done Done Done false 1002 Acked Read 3 Done Done Done false 1003 Acked Read 4 Done Done Done false 1004 Acked Read 8 Done Done Done false 1005 Acked Write 10 Sent Done Done false 1006 Acked Write 3 Sent Done Done false 1007 Acked Read 4 Sent Done Done false 1008 Acked Read 1 Sent Done Done false 1009 Acked Write 2 Sent Done Done false 1010 Acked Read 10 Sent Done Done false 1011 Acked Read 8 Sent Done Done false 1012 Acked Write 4 Sent Done Done false 1013 Acked Read 9 Sent Done Done false 1014 Acked Flush 0 Sent Done Done false 1015 Acked Write 4 Sent Done Done false 1016 Acked Read 8 Sent Done Done false 1017 Acked Read 6 Sent Done Done false 1018 Acked Write 8 Sent Done Done false 1019 Acked Write 5 Sent Done Done false 1020 Acked Read 2 Sent Done Done false 1021 Acked Write 8 Sent Done Done false 1022 Acked Read 4 Sent Done Done false 1023 Acked Write 10 Sent Done Done false 1024 Acked Read 2 Sent Done Done false 1025 Acked Read 2 Sent Done Done false 1026 Acked Read 9 Sent Done Done false 1027 Acked Write 5 Sent Sent Sent false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 25 3 3 31 Done 4 26 26 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1000)) Some(JobId(1014)) Some(JobId(1014)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:37.059982157Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1256} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:37.060157408Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1256,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 23 23 23 33 33 33 F F F 1 020-039 27 27 27 37 37 37 F F F 2 040-059 29 29 29 39 39 39 F F F 3 060-079 22 22 22 32 32 32 F F F 4 080-099 29 29 29 39 39 39 F T T <--- 5 100-119 27 27 27 37 37 37 F F F 6 120-139 29 29 29 39 39 39 F F F 7 140-159 27 27 27 37 37 37 F T T <--- 8 160-179 26 26 26 36 36 36 F F F 9 180-199 29 30 30 39 41 41 T T T <--- 10 200-219 29 30 30 39 41 41 F T T <--- 11 220-239 18 18 18 27 27 27 F T T <--- 12 240-259 24 24 24 34 34 34 F T T <--- 13 260-279 24 24 24 34 34 34 F F F 14 280-299 28 28 28 38 38 38 F F F 15 300-319 18 30 30 27 41 41 F F F <--- 16 320-339 29 29 29 39 39 39 F F F 17 340-359 27 27 27 37 37 37 F F F 18 360-379 29 29 29 39 39 39 F F F 19 380-399 29 29 29 39 39 39 F F F 20 400-419 29 29 29 39 39 39 F F F 21 420-439 29 29 29 39 39 39 F F F 22 440-459 27 30 30 37 41 41 F T T <--- 23 460-479 21 21 21 30 30 30 F F F 24 480-499 27 27 27 37 37 37 F F F 25 500-519 29 30 30 39 41 41 F F F <--- 26 520-539 29 30 30 39 41 41 F T T <--- 27 540-559 28 28 28 38 38 38 F F F 28 560-579 28 28 28 38 38 38 F F F 29 580-599 22 22 22 32 32 32 F T T <--- Max gen: 30, Max flush: 41 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 15 Verifying data now No change after verify Loop: 15 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 23 23 23 33 33 33 F F F 1 020-039 27 27 27 37 37 37 F F F 2 040-059 29 29 29 39 39 39 F F F 3 060-079 22 22 22 32 32 32 F F F 4 080-099 31 31 31 42 42 42 F F F 5 100-119 27 27 27 37 37 37 F F F 6 120-139 29 29 29 39 39 39 F F F 7 140-159 31 31 31 42 42 42 F F F 8 160-179 26 26 26 36 36 36 F F F 9 180-199 31 31 31 42 42 42 F F F 10 200-219 31 31 31 42 42 42 F F F 11 220-239 31 31 31 42 42 42 F F F 12 240-259 31 31 31 42 42 42 F F F 13 260-279 24 24 24 34 34 34 F F F 14 280-299 28 28 28 38 38 38 F F F 15 300-319 30 30 30 41 41 41 F F F 16 320-339 29 29 29 39 39 39 F F F 17 340-359 27 27 27 37 37 37 F F F 18 360-379 29 29 29 39 39 39 F F F 19 380-399 29 29 29 39 39 39 F F F 20 400-419 29 29 29 39 39 39 F F F 21 420-439 29 29 29 39 39 39 F F F 22 440-459 31 31 31 42 42 42 F F F 23 460-479 21 21 21 30 30 30 F F F 24 480-499 27 27 27 37 37 37 F F F 25 500-519 30 30 30 41 41 41 F F F 26 520-539 31 31 31 42 42 42 F F F 27 540-559 28 28 28 38 38 38 F F F 28 560-579 28 28 28 38 38 38 F F F 29 580-599 31 31 31 42 42 42 F F F Max gen: 31, Max flush: 42 Begin loop 16 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:41.312508894Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1264,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:41.320722927Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1264,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:41.321017447Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1264} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:41.321045077Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1264} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:41.321055507Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1264} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:41.321067307Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1264} {"msg":"Crucible 81092528-a472-4e19-9e55-214dda2abd7a has session id: b1e94fe5-1dfc-48e1-8f75-5f995781b5d4","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:41.321440228Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1264,"session_id":"b1e94fe5-1dfc-48e1-8f75-5f995781b5d4"} {"msg":"Upstairs opts: Upstairs UUID: 81092528-a472-4e19-9e55-214dda2abd7a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:41.321463048Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1264,"session_id":"b1e94fe5-1dfc-48e1-8f75-5f995781b5d4"} {"msg":"Crucible stats registered with UUID: 81092528-a472-4e19-9e55-214dda2abd7a","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:41.321473318Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1264,"session_id":"b1e94fe5-1dfc-48e1-8f75-5f995781b5d4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:41.321482628Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1264,"session_id":"b1e94fe5-1dfc-48e1-8f75-5f995781b5d4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:41.321627078Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1264,"":"io task","client":"0","":"downstairs","session_id":"b1e94fe5-1dfc-48e1-8f75-5f995781b5d4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:41.321644998Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1264,"":"io task","client":"1","":"downstairs","session_id":"b1e94fe5-1dfc-48e1-8f75-5f995781b5d4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:41.321655208Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1264,"":"io task","client":"2","":"downstairs","session_id":"b1e94fe5-1dfc-48e1-8f75-5f995781b5d4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:41.321797248Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1264,"task":"crutest"} {"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:41.321813878Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1264} {"msg":"81092528-a472-4e19-9e55-214dda2abd7a active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:41.321846369Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1264,"session_id":"b1e94fe5-1dfc-48e1-8f75-5f995781b5d4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:41.321855919Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1264,"client":"0","":"downstairs","session_id":"b1e94fe5-1dfc-48e1-8f75-5f995781b5d4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:41.321865769Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1264,"client":"1","":"downstairs","session_id":"b1e94fe5-1dfc-48e1-8f75-5f995781b5d4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:41.321876819Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1264,"client":"2","":"downstairs","session_id":"b1e94fe5-1dfc-48e1-8f75-5f995781b5d4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:41.321886599Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1264,"":"io task","client":"2","":"downstairs","session_id":"b1e94fe5-1dfc-48e1-8f75-5f995781b5d4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:41.321896639Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1264,"":"io task","client":"0","":"downstairs","session_id":"b1e94fe5-1dfc-48e1-8f75-5f995781b5d4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:41.321906589Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1264,"":"io task","client":"1","":"downstairs","session_id":"b1e94fe5-1dfc-48e1-8f75-5f995781b5d4"} {"msg":"ds_connection connected from Ok(127.0.0.1:41972)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:41.321965639Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1264,"":"io task","client":"2","":"downstairs","session_id":"b1e94fe5-1dfc-48e1-8f75-5f995781b5d4"} {"msg":"ds_connection connected from Ok(127.0.0.1:44325)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:41.321991139Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1264,"":"io task","client":"0","":"downstairs","session_id":"b1e94fe5-1dfc-48e1-8f75-5f995781b5d4"} {"msg":"ds_connection connected from Ok(127.0.0.1:51422)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:41.322002319Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1264,"":"io task","client":"1","":"downstairs","session_id":"b1e94fe5-1dfc-48e1-8f75-5f995781b5d4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:41.32270848Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1264,"client":"1","":"downstairs","session_id":"b1e94fe5-1dfc-48e1-8f75-5f995781b5d4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:41.3227573Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1264,"client":"2","":"downstairs","session_id":"b1e94fe5-1dfc-48e1-8f75-5f995781b5d4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:41.32276947Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1264,"client":"0","":"downstairs","session_id":"b1e94fe5-1dfc-48e1-8f75-5f995781b5d4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:41.32305603Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1264,"session_id":"b1e94fe5-1dfc-48e1-8f75-5f995781b5d4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:41.32310077Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1264,"session_id":"b1e94fe5-1dfc-48e1-8f75-5f995781b5d4"} {"msg":"[0]R flush_numbers[0..12]: [33, 37, 39, 32, 42, 37, 39, 42, 36, 42, 42, 42]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:41.323183191Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1264,"":"downstairs","":"downstairs","session_id":"b1e94fe5-1dfc-48e1-8f75-5f995781b5d4"} {"msg":"[0]R generation[0..12]: [23, 27, 29, 22, 31, 27, 29, 31, 26, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:41.323194731Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1264,"":"downstairs","":"downstairs","session_id":"b1e94fe5-1dfc-48e1-8f75-5f995781b5d4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:41.323208231Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1264,"":"downstairs","":"downstairs","session_id":"b1e94fe5-1dfc-48e1-8f75-5f995781b5d4"} {"msg":"[1]R flush_numbers[0..12]: [33, 37, 39, 32, 42, 37, 39, 42, 36, 42, 42, 42]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:41.323218941Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1264,"":"downstairs","":"downstairs","session_id":"b1e94fe5-1dfc-48e1-8f75-5f995781b5d4"} {"msg":"[1]R generation[0..12]: [23, 27, 29, 22, 31, 27, 29, 31, 26, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:41.323229451Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1264,"":"downstairs","":"downstairs","session_id":"b1e94fe5-1dfc-48e1-8f75-5f995781b5d4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:41.323240321Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1264,"":"downstairs","":"downstairs","session_id":"b1e94fe5-1dfc-48e1-8f75-5f995781b5d4"} {"msg":"[2]R flush_numbers[0..12]: [33, 37, 39, 32, 42, 37, 39, 42, 36, 42, 42, 42]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:41.323250821Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1264,"":"downstairs","":"downstairs","session_id":"b1e94fe5-1dfc-48e1-8f75-5f995781b5d4"} {"msg":"[2]R generation[0..12]: [23, 27, 29, 22, 31, 27, 29, 31, 26, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:41.323261261Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1264,"":"downstairs","":"downstairs","session_id":"b1e94fe5-1dfc-48e1-8f75-5f995781b5d4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:41.323272061Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1264,"":"downstairs","":"downstairs","session_id":"b1e94fe5-1dfc-48e1-8f75-5f995781b5d4"} {"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:41.323281631Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1264,"":"downstairs","":"downstairs","session_id":"b1e94fe5-1dfc-48e1-8f75-5f995781b5d4"} {"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:41.323291511Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1264,"":"downstairs","":"downstairs","session_id":"b1e94fe5-1dfc-48e1-8f75-5f995781b5d4"} {"msg":"Next flush: 43","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:41.323301051Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1264,"":"downstairs","":"downstairs","session_id":"b1e94fe5-1dfc-48e1-8f75-5f995781b5d4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:41.323310581Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1264,"":"downstairs","":"downstairs","session_id":"b1e94fe5-1dfc-48e1-8f75-5f995781b5d4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:41.323320491Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1264,"session_id":"b1e94fe5-1dfc-48e1-8f75-5f995781b5d4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:41.323329351Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1264,"client":"0","":"downstairs","session_id":"b1e94fe5-1dfc-48e1-8f75-5f995781b5d4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:41.323339081Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1264,"client":"1","":"downstairs","session_id":"b1e94fe5-1dfc-48e1-8f75-5f995781b5d4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:41.323348761Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1264,"client":"2","":"downstairs","session_id":"b1e94fe5-1dfc-48e1-8f75-5f995781b5d4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:41.323359871Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1264,"session_id":"b1e94fe5-1dfc-48e1-8f75-5f995781b5d4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:41.323369201Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1264,"session_id":"b1e94fe5-1dfc-48e1-8f75-5f995781b5d4"} {"msg":"81092528-a472-4e19-9e55-214dda2abd7a is now active with session: b1e94fe5-1dfc-48e1-8f75-5f995781b5d4","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:41.323379321Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1264,"session_id":"b1e94fe5-1dfc-48e1-8f75-5f995781b5d4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:41.323388211Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1264,"session_id":"b1e94fe5-1dfc-48e1-8f75-5f995781b5d4"} {"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:41.323397481Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1264} {"msg":"Activated sub_volume 81092528-a472-4e19-9e55-214dda2abd7a","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:41.323405821Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1264} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:41.323414261Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1264,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:41.323699441Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1264,"task":"crutest"} 01/30 Write block 314 len 4096 data: 2 02/30 Read block 69 len 28672 03/30 Read block 216 len 16384 04/30 Write block 348 len 16384 data: 4 3 2 2 05/30 Write block 543 len 40960 data: 3 3 2 2 2 3 3 2 2 3 06/30 Write block 268 len 20480 data: 6 7 5 5 6 07/30 Write block 180 len 24576 data: 3 4 3 3 3 5 08/30 Write block 385 len 32768 data: 4 3 3 5 6 5 4 4 09/30 Read block 564 len 36864 10/30 Write block 15 len 36864 data: 3 4 4 5 7 8 8 7 7 11/30 Read block 206 len 8192 12/30 Write block 572 len 8192 data: 5 3 13/30 Write block 207 len 20480 data: 3 4 5 5 5 14/30 Write block 207 len 24576 data: 4 5 6 6 6 6 15/30 Flush 16/30 Write block 113 len 16384 data: 5 4 3 3 17/30 Write block 542 len 20480 data: 4 4 4 3 3 18/30 Read block 181 len 40960 19/30 Read block 392 len 32768 20/30 Read block 320 len 36864 21/30 Write block 378 len 40960 data: 6 5 5 5 4 4 3 5 4 4 22/30 Flush 23/30 Write block 139 len 12288 data: 6 5 5 24/30 Read block 36 len 32768 25/30 Write block 416 len 12288 data: 4 4 4 26/30 Read block 392 len 24576 27/30 Read block 399 len 32768 28/30 Read block 394 len 32768 29/30 Read block 39 len 36864 30/30 Write block 50 len 36864 data: 5 6 6 5 3 3 3 3 4 ---------------------------------------------------------------- Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Read 7 Done Done Done false 1002 Acked Read 4 Done Done Done false 1003 Acked Write 4 Done Sent Done false 1004 Acked Write 10 Done Sent Done false 1005 Acked Write 5 Done Sent Done false 1006 Acked Write 6 Done Sent Done false 1007 Acked Write 8 Done Sent Done false 1008 Acked Read 9 Done Sent Done false 1009 Acked Write 9 Done Sent Done false 1010 Acked Read 2 Done Sent Done false 1011 Acked Write 2 Done Sent Done false 1012 Acked Write 5 Done Sent Done false 1013 Acked Write 6 Done Sent Done false 1014 Acked Flush 0 Done Sent Done false 1015 Acked Write 4 Done Sent Done false 1016 Acked Write 5 Done Sent Done false 1017 Acked Read 10 Done Sent Done false 1018 Acked Read 8 Done Sent Done false 1019 Acked Read 9 Done Sent Done false 1020 Acked Write 10 Done Sent Done false 1021 Acked Flush 0 Done Sent Done false 1022 Acked Write 3 Done Sent Done false 1023 Acked Read 8 Done Sent Done false 1024 Acked Write 3 Done Sent Done false 1025 Acked Read 6 Done Sent Done false 1026 Acked Read 8 Done Sent Done false 1027 Acked Read 8 Done Sent Done false 1028 Acked Read 9 Done Sent Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 27 1 29 Done 29 3 29 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) None Some(JobId(1021)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:41.332462905Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1264} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:41.332644715Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1264,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 32 23 32 43 33 43 F F F <--- 1 020-039 32 27 32 43 37 43 F F F <--- 2 040-059 29 29 29 39 39 39 T F T <--- 3 060-079 22 22 22 32 32 32 F F F 4 080-099 31 31 31 42 42 42 F F F 5 100-119 32 27 32 44 37 44 F F F <--- 6 120-139 29 29 29 39 39 39 T F T <--- 7 140-159 31 31 31 42 42 42 T F T <--- 8 160-179 26 26 26 36 36 36 F F F 9 180-199 32 31 32 43 42 43 F F F <--- 10 200-219 32 31 32 43 42 43 F F F <--- 11 220-239 31 31 31 42 42 42 F F F 12 240-259 31 31 31 42 42 42 F F F 13 260-279 32 24 32 43 34 43 F F F <--- 14 280-299 28 28 28 38 38 38 F F F 15 300-319 32 30 32 43 41 43 F T F <--- 16 320-339 29 29 29 39 39 39 F F F 17 340-359 32 27 32 43 37 43 F F F <--- 18 360-379 32 29 32 44 39 44 F F F <--- 19 380-399 32 29 32 44 39 44 F F F <--- 20 400-419 29 29 29 39 39 39 T F T <--- 21 420-439 29 29 29 39 39 39 F F F 22 440-459 31 31 31 42 42 42 F F F 23 460-479 21 21 21 30 30 30 F F F 24 480-499 27 27 27 37 37 37 F F F 25 500-519 30 30 30 41 41 41 F F F 26 520-539 31 31 31 42 42 42 F F F 27 540-559 32 28 32 44 38 44 F F F <--- 28 560-579 32 28 32 43 38 43 F F F <--- 29 580-599 31 31 31 42 42 42 F F F Max gen: 32, Max flush: 44 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 16 Verifying data now No change after verify Loop: 16 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 32 32 32 43 43 43 F F F 1 020-039 32 32 32 43 43 43 F F F 2 040-059 33 33 33 45 45 45 F F F 3 060-079 22 22 22 32 32 32 F F F 4 080-099 31 31 31 42 42 42 F F F 5 100-119 32 32 32 44 44 44 F F F 6 120-139 33 33 33 45 45 45 F F F 7 140-159 33 33 33 45 45 45 F F F 8 160-179 26 26 26 36 36 36 F F F 9 180-199 32 32 32 43 43 43 F F F 10 200-219 32 32 32 43 43 43 F F F 11 220-239 31 31 31 42 42 42 F F F 12 240-259 31 31 31 42 42 42 F F F 13 260-279 32 32 32 43 43 43 F F F 14 280-299 28 28 28 38 38 38 F F F 15 300-319 32 32 32 43 43 43 F F F 16 320-339 29 29 29 39 39 39 F F F 17 340-359 32 32 32 43 43 43 F F F 18 360-379 32 32 32 44 44 44 F F F 19 380-399 32 32 32 44 44 44 F F F 20 400-419 33 33 33 45 45 45 F F F 21 420-439 29 29 29 39 39 39 F F F 22 440-459 31 31 31 42 42 42 F F F 23 460-479 21 21 21 30 30 30 F F F 24 480-499 27 27 27 37 37 37 F F F 25 500-519 30 30 30 41 41 41 F F F 26 520-539 31 31 31 42 42 42 F F F 27 540-559 32 32 32 44 44 44 F F F 28 560-579 32 32 32 43 43 43 F F F 29 580-599 31 31 31 42 42 42 F F F Max gen: 33, Max flush: 45 Begin loop 17 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:45.586549384Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1272,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:45.594604266Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1272,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:45.594895217Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1272} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:45.594922567Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1272} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:45.594931267Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1272} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:45.594943167Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1272} {"msg":"Crucible c05fe6de-6271-42e5-be99-53f568445d87 has session id: e20057bf-8b97-4195-aac3-dc6441c7e162","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:45.595344237Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1272,"session_id":"e20057bf-8b97-4195-aac3-dc6441c7e162"} {"msg":"Upstairs opts: Upstairs UUID: c05fe6de-6271-42e5-be99-53f568445d87, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:45.595373587Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1272,"session_id":"e20057bf-8b97-4195-aac3-dc6441c7e162"} {"msg":"Crucible stats registered with UUID: c05fe6de-6271-42e5-be99-53f568445d87","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:45.595387327Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1272,"session_id":"e20057bf-8b97-4195-aac3-dc6441c7e162"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:45.595396737Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1272,"session_id":"e20057bf-8b97-4195-aac3-dc6441c7e162"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:45.595524998Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1272,"":"io task","client":"0","":"downstairs","session_id":"e20057bf-8b97-4195-aac3-dc6441c7e162"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:45.595542578Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1272,"":"io task","client":"2","":"downstairs","session_id":"e20057bf-8b97-4195-aac3-dc6441c7e162"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:45.595552688Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1272,"":"io task","client":"1","":"downstairs","session_id":"e20057bf-8b97-4195-aac3-dc6441c7e162"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:45.595716218Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1272,"task":"crutest"} {"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:45.595728618Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1272} {"msg":"c05fe6de-6271-42e5-be99-53f568445d87 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:45.595745268Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1272,"session_id":"e20057bf-8b97-4195-aac3-dc6441c7e162"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:45.595754288Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1272,"client":"0","":"downstairs","session_id":"e20057bf-8b97-4195-aac3-dc6441c7e162"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:45.595764008Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1272,"client":"1","":"downstairs","session_id":"e20057bf-8b97-4195-aac3-dc6441c7e162"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:45.595773878Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1272,"client":"2","":"downstairs","session_id":"e20057bf-8b97-4195-aac3-dc6441c7e162"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:45.595783508Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1272,"":"io task","client":"2","":"downstairs","session_id":"e20057bf-8b97-4195-aac3-dc6441c7e162"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:45.595793408Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1272,"":"io task","client":"0","":"downstairs","session_id":"e20057bf-8b97-4195-aac3-dc6441c7e162"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:45.595803458Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1272,"":"io task","client":"1","":"downstairs","session_id":"e20057bf-8b97-4195-aac3-dc6441c7e162"} {"msg":"ds_connection connected from Ok(127.0.0.1:63948)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:45.595871338Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1272,"":"io task","client":"0","":"downstairs","session_id":"e20057bf-8b97-4195-aac3-dc6441c7e162"} {"msg":"ds_connection connected from Ok(127.0.0.1:36968)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:45.595883978Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1272,"":"io task","client":"2","":"downstairs","session_id":"e20057bf-8b97-4195-aac3-dc6441c7e162"} {"msg":"ds_connection connected from Ok(127.0.0.1:32981)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:45.595894198Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1272,"":"io task","client":"1","":"downstairs","session_id":"e20057bf-8b97-4195-aac3-dc6441c7e162"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:45.596592739Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1272,"client":"0","":"downstairs","session_id":"e20057bf-8b97-4195-aac3-dc6441c7e162"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:45.596617779Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1272,"client":"2","":"downstairs","session_id":"e20057bf-8b97-4195-aac3-dc6441c7e162"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:45.596711089Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1272,"client":"1","":"downstairs","session_id":"e20057bf-8b97-4195-aac3-dc6441c7e162"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:45.5969287Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1272,"session_id":"e20057bf-8b97-4195-aac3-dc6441c7e162"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:45.59695273Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1272,"session_id":"e20057bf-8b97-4195-aac3-dc6441c7e162"} {"msg":"[0]R flush_numbers[0..12]: [43, 43, 45, 32, 42, 44, 45, 45, 36, 43, 43, 42]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:45.59702839Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1272,"":"downstairs","":"downstairs","session_id":"e20057bf-8b97-4195-aac3-dc6441c7e162"} {"msg":"[0]R generation[0..12]: [32, 32, 33, 22, 31, 32, 33, 33, 26, 32, 32, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:45.59704947Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1272,"":"downstairs","":"downstairs","session_id":"e20057bf-8b97-4195-aac3-dc6441c7e162"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:45.59706096Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1272,"":"downstairs","":"downstairs","session_id":"e20057bf-8b97-4195-aac3-dc6441c7e162"} {"msg":"[1]R flush_numbers[0..12]: [43, 43, 45, 32, 42, 44, 45, 45, 36, 43, 43, 42]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:45.59707161Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1272,"":"downstairs","":"downstairs","session_id":"e20057bf-8b97-4195-aac3-dc6441c7e162"} {"msg":"[1]R generation[0..12]: [32, 32, 33, 22, 31, 32, 33, 33, 26, 32, 32, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:45.5970821Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1272,"":"downstairs","":"downstairs","session_id":"e20057bf-8b97-4195-aac3-dc6441c7e162"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:45.59709369Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1272,"":"downstairs","":"downstairs","session_id":"e20057bf-8b97-4195-aac3-dc6441c7e162"} {"msg":"[2]R flush_numbers[0..12]: [43, 43, 45, 32, 42, 44, 45, 45, 36, 43, 43, 42]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:45.59710427Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1272,"":"downstairs","":"downstairs","session_id":"e20057bf-8b97-4195-aac3-dc6441c7e162"} {"msg":"[2]R generation[0..12]: [32, 32, 33, 22, 31, 32, 33, 33, 26, 32, 32, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:45.59711713Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1272,"":"downstairs","":"downstairs","session_id":"e20057bf-8b97-4195-aac3-dc6441c7e162"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:45.59712842Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1272,"":"downstairs","":"downstairs","session_id":"e20057bf-8b97-4195-aac3-dc6441c7e162"} {"msg":"Max found gen is 34","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:45.59713799Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1272,"":"downstairs","":"downstairs","session_id":"e20057bf-8b97-4195-aac3-dc6441c7e162"} {"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:45.59714787Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1272,"":"downstairs","":"downstairs","session_id":"e20057bf-8b97-4195-aac3-dc6441c7e162"} {"msg":"Next flush: 46","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:45.59715728Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1272,"":"downstairs","":"downstairs","session_id":"e20057bf-8b97-4195-aac3-dc6441c7e162"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:45.59716669Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1272,"":"downstairs","":"downstairs","session_id":"e20057bf-8b97-4195-aac3-dc6441c7e162"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:45.59717646Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1272,"session_id":"e20057bf-8b97-4195-aac3-dc6441c7e162"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:45.59718518Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1272,"client":"0","":"downstairs","session_id":"e20057bf-8b97-4195-aac3-dc6441c7e162"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:45.59719479Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1272,"client":"1","":"downstairs","session_id":"e20057bf-8b97-4195-aac3-dc6441c7e162"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:45.59720443Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1272,"client":"2","":"downstairs","session_id":"e20057bf-8b97-4195-aac3-dc6441c7e162"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:45.59721439Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1272,"session_id":"e20057bf-8b97-4195-aac3-dc6441c7e162"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:45.59722374Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1272,"session_id":"e20057bf-8b97-4195-aac3-dc6441c7e162"} {"msg":"c05fe6de-6271-42e5-be99-53f568445d87 is now active with session: e20057bf-8b97-4195-aac3-dc6441c7e162","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:45.59723376Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1272,"session_id":"e20057bf-8b97-4195-aac3-dc6441c7e162"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:45.59724286Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1272,"session_id":"e20057bf-8b97-4195-aac3-dc6441c7e162"} {"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:45.59725197Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1272} {"msg":"Activated sub_volume c05fe6de-6271-42e5-be99-53f568445d87","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:45.59726002Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1272} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:45.59726829Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1272,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:45.597548281Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1272,"task":"crutest"} 01/30 Write block 270 len 4096 data: 6 02/30 Read block 452 len 36864 03/30 Write block 442 len 36864 data: 2 2 2 2 2 3 4 4 3 04/30 Write block 138 len 32768 data: 5 7 6 6 4 3 4 3 05/30 Read block 23 len 36864 06/30 Read block 482 len 16384 07/30 Write block 554 len 12288 data: 2 3 3 08/30 Read block 470 len 20480 09/30 Write block 183 len 24576 data: 4 4 6 5 7 7 10/30 Write block 383 len 24576 data: 5 4 6 5 5 6 11/30 Write block 60 len 8192 data: 4 5 12/30 Write block 265 len 28672 data: 5 6 6 7 8 7 6 13/30 Read block 587 len 12288 14/30 Write block 480 len 32768 data: 3 2 2 2 3 3 3 3 15/30 Read block 418 len 28672 16/30 Read block 276 len 16384 17/30 Read block 479 len 16384 18/30 Read block 15 len 24576 19/30 Read block 41 len 32768 20/30 Write block 203 len 20480 data: 4 4 4 3 5 21/30 Write block 133 len 36864 data: 2 2 3 3 4 6 8 7 7 22/30 Read block 416 len 40960 23/30 Read block 508 len 4096 24/30 Write block 132 len 4096 data: 2 25/30 Write block 527 len 4096 data: 6 26/30 Write block 463 len 8192 data: 5 4 27/30 Write block 202 len 16384 data: 3 5 5 5 28/30 Write block 28 len 8192 data: 2 2 29/30 Write block 178 len 12288 data: 4 3 4 30/30 Write block 116 len 28672 data: 4 2 3 3 3 4 4 ---------------------------------------------------------------- Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Read 9 Done Done Done false 1002 Acked Write 9 Done Sent Done false 1003 Acked Write 8 Done Sent Done false 1004 Acked Read 9 Done Sent Done false 1005 Acked Read 4 Done Sent Done false 1006 Acked Write 3 Done Sent Done false 1007 Acked Read 5 Done Sent Done false 1008 Acked Write 6 Done Sent Done false 1009 Acked Write 6 Done Sent Done false 1010 Acked Write 2 Done Sent Done false 1011 Acked Write 7 Done Sent Done false 1012 Acked Read 3 Done Sent Done false 1013 Acked Write 8 Done Sent Done false 1014 Acked Read 7 Done Sent Done false 1015 Acked Read 4 Done Sent Done false 1016 Acked Read 4 Done Sent Done false 1017 Acked Read 6 Done Sent Done false 1018 Acked Read 8 Done Sent Done false 1019 Acked Write 5 Done Sent Done false 1020 Acked Write 9 Done Sent Done false 1021 Acked Read 10 Done Sent Done false 1022 Acked Read 1 Done Sent Done false 1023 Acked Write 1 Done Sent Done false 1024 Acked Write 1 Done Sent Done false 1025 Acked Write 2 Done Sent Done false 1026 Acked Write 4 Sent Sent Done false 1027 Acked Write 2 Sent Sent Done false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 4 28 2 34 Done 26 2 28 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:45.605026632Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1272} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:45.605212562Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1272,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 32 32 32 43 43 43 F F F 1 020-039 32 32 32 43 43 43 T F T <--- 2 040-059 33 33 33 45 45 45 F F F 3 060-079 22 22 22 32 32 32 T F T <--- 4 080-099 31 31 31 42 42 42 F F F 5 100-119 32 32 32 44 44 44 T F T <--- 6 120-139 33 33 33 45 45 45 T F T <--- 7 140-159 33 33 33 45 45 45 T F T <--- 8 160-179 26 26 26 36 36 36 T F T <--- 9 180-199 32 32 32 43 43 43 T F T <--- 10 200-219 32 32 32 43 43 43 T F T <--- 11 220-239 31 31 31 42 42 42 F F F 12 240-259 31 31 31 42 42 42 F F F 13 260-279 32 32 32 43 43 43 T T T 14 280-299 28 28 28 38 38 38 F F F 15 300-319 32 32 32 43 43 43 F F F 16 320-339 29 29 29 39 39 39 F F F 17 340-359 32 32 32 43 43 43 F F F 18 360-379 32 32 32 44 44 44 F F F 19 380-399 32 32 32 44 44 44 T F T <--- 20 400-419 33 33 33 45 45 45 F F F 21 420-439 29 29 29 39 39 39 F F F 22 440-459 31 31 31 42 42 42 T F T <--- 23 460-479 21 21 21 30 30 30 T F T <--- 24 480-499 27 27 27 37 37 37 T F T <--- 25 500-519 30 30 30 41 41 41 F F F 26 520-539 31 31 31 42 42 42 T F T <--- 27 540-559 32 32 32 44 44 44 T F T <--- 28 560-579 32 32 32 43 43 43 F F F 29 580-599 31 31 31 42 42 42 F F F Max gen: 33, Max flush: 45 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 17 Verifying data now No change after verify Loop: 17 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 32 32 32 43 43 43 F F F 1 020-039 34 34 34 46 46 46 F F F 2 040-059 33 33 33 45 45 45 F F F 3 060-079 34 34 34 46 46 46 F F F 4 080-099 31 31 31 42 42 42 F F F 5 100-119 34 34 34 46 46 46 F F F 6 120-139 34 34 34 46 46 46 F F F 7 140-159 34 34 34 46 46 46 F F F 8 160-179 34 34 34 46 46 46 F F F 9 180-199 34 34 34 46 46 46 F F F 10 200-219 34 34 34 46 46 46 F F F 11 220-239 31 31 31 42 42 42 F F F 12 240-259 31 31 31 42 42 42 F F F 13 260-279 34 34 34 46 46 46 F F F 14 280-299 28 28 28 38 38 38 F F F 15 300-319 32 32 32 43 43 43 F F F 16 320-339 29 29 29 39 39 39 F F F 17 340-359 32 32 32 43 43 43 F F F 18 360-379 32 32 32 44 44 44 F F F 19 380-399 34 34 34 46 46 46 F F F 20 400-419 33 33 33 45 45 45 F F F 21 420-439 29 29 29 39 39 39 F F F 22 440-459 34 34 34 46 46 46 F F F 23 460-479 34 34 34 46 46 46 F F F 24 480-499 34 34 34 46 46 46 F F F 25 500-519 30 30 30 41 41 41 F F F 26 520-539 34 34 34 46 46 46 F F F 27 540-559 34 34 34 46 46 46 F F F 28 560-579 32 32 32 43 43 43 F F F 29 580-599 31 31 31 42 42 42 F F F Max gen: 34, Max flush: 46 Begin loop 18 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:49.865562561Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1280,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:49.873697163Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1280,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:49.873990244Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1280} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:49.874017994Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1280} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:49.874026604Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1280} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:49.874038514Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1280} {"msg":"Crucible e01fa7c1-7e63-4c69-a9ea-bfadb5ccfd64 has session id: a06228ef-19ae-435b-b1cb-98caf5aab414","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:49.874388974Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1280,"session_id":"a06228ef-19ae-435b-b1cb-98caf5aab414"} {"msg":"Upstairs opts: Upstairs UUID: e01fa7c1-7e63-4c69-a9ea-bfadb5ccfd64, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:49.874412754Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1280,"session_id":"a06228ef-19ae-435b-b1cb-98caf5aab414"} {"msg":"Crucible stats registered with UUID: e01fa7c1-7e63-4c69-a9ea-bfadb5ccfd64","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:49.874422694Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1280,"session_id":"a06228ef-19ae-435b-b1cb-98caf5aab414"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:49.874431974Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1280,"session_id":"a06228ef-19ae-435b-b1cb-98caf5aab414"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:49.874575084Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1280,"":"io task","client":"1","":"downstairs","session_id":"a06228ef-19ae-435b-b1cb-98caf5aab414"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:49.874607394Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1280,"":"io task","client":"0","":"downstairs","session_id":"a06228ef-19ae-435b-b1cb-98caf5aab414"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:49.874618744Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1280,"":"io task","client":"2","":"downstairs","session_id":"a06228ef-19ae-435b-b1cb-98caf5aab414"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:49.874753935Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1280,"task":"crutest"} {"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:49.874773035Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1280} {"msg":"e01fa7c1-7e63-4c69-a9ea-bfadb5ccfd64 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:49.874789485Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1280,"session_id":"a06228ef-19ae-435b-b1cb-98caf5aab414"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:49.874798405Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1280,"client":"0","":"downstairs","session_id":"a06228ef-19ae-435b-b1cb-98caf5aab414"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:49.874808075Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1280,"client":"1","":"downstairs","session_id":"a06228ef-19ae-435b-b1cb-98caf5aab414"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:49.874817785Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1280,"client":"2","":"downstairs","session_id":"a06228ef-19ae-435b-b1cb-98caf5aab414"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:49.874827425Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1280,"":"io task","client":"2","":"downstairs","session_id":"a06228ef-19ae-435b-b1cb-98caf5aab414"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:49.874837365Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1280,"":"io task","client":"0","":"downstairs","session_id":"a06228ef-19ae-435b-b1cb-98caf5aab414"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:49.874847345Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1280,"":"io task","client":"1","":"downstairs","session_id":"a06228ef-19ae-435b-b1cb-98caf5aab414"} {"msg":"ds_connection connected from Ok(127.0.0.1:60925)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:49.874891555Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1280,"":"io task","client":"2","":"downstairs","session_id":"a06228ef-19ae-435b-b1cb-98caf5aab414"} {"msg":"ds_connection connected from Ok(127.0.0.1:48941)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:49.874902545Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1280,"":"io task","client":"1","":"downstairs","session_id":"a06228ef-19ae-435b-b1cb-98caf5aab414"} {"msg":"ds_connection connected from Ok(127.0.0.1:39683)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:49.874912835Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1280,"":"io task","client":"0","":"downstairs","session_id":"a06228ef-19ae-435b-b1cb-98caf5aab414"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:49.875495746Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1280,"client":"0","":"downstairs","session_id":"a06228ef-19ae-435b-b1cb-98caf5aab414"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:49.875532526Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1280,"client":"1","":"downstairs","session_id":"a06228ef-19ae-435b-b1cb-98caf5aab414"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:49.875672986Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1280,"client":"2","":"downstairs","session_id":"a06228ef-19ae-435b-b1cb-98caf5aab414"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:49.875819156Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1280,"session_id":"a06228ef-19ae-435b-b1cb-98caf5aab414"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:49.875854936Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1280,"session_id":"a06228ef-19ae-435b-b1cb-98caf5aab414"} {"msg":"[0]R flush_numbers[0..12]: [43, 46, 45, 46, 42, 46, 46, 46, 46, 46, 46, 42]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:49.875985747Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1280,"":"downstairs","":"downstairs","session_id":"a06228ef-19ae-435b-b1cb-98caf5aab414"} {"msg":"[0]R generation[0..12]: [32, 34, 33, 34, 31, 34, 34, 34, 34, 34, 34, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:49.875996847Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1280,"":"downstairs","":"downstairs","session_id":"a06228ef-19ae-435b-b1cb-98caf5aab414"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:49.876008087Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1280,"":"downstairs","":"downstairs","session_id":"a06228ef-19ae-435b-b1cb-98caf5aab414"} {"msg":"[1]R flush_numbers[0..12]: [43, 46, 45, 46, 42, 46, 46, 46, 46, 46, 46, 42]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:49.876018567Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1280,"":"downstairs","":"downstairs","session_id":"a06228ef-19ae-435b-b1cb-98caf5aab414"} {"msg":"[1]R generation[0..12]: [32, 34, 33, 34, 31, 34, 34, 34, 34, 34, 34, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:49.876028887Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1280,"":"downstairs","":"downstairs","session_id":"a06228ef-19ae-435b-b1cb-98caf5aab414"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:49.876039867Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1280,"":"downstairs","":"downstairs","session_id":"a06228ef-19ae-435b-b1cb-98caf5aab414"} {"msg":"[2]R flush_numbers[0..12]: [43, 46, 45, 46, 42, 46, 46, 46, 46, 46, 46, 42]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:49.876050497Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1280,"":"downstairs","":"downstairs","session_id":"a06228ef-19ae-435b-b1cb-98caf5aab414"} {"msg":"[2]R generation[0..12]: [32, 34, 33, 34, 31, 34, 34, 34, 34, 34, 34, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:49.876060897Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1280,"":"downstairs","":"downstairs","session_id":"a06228ef-19ae-435b-b1cb-98caf5aab414"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:49.876071807Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1280,"":"downstairs","":"downstairs","session_id":"a06228ef-19ae-435b-b1cb-98caf5aab414"} {"msg":"Max found gen is 35","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:49.876081257Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1280,"":"downstairs","":"downstairs","session_id":"a06228ef-19ae-435b-b1cb-98caf5aab414"} {"msg":"Generation requested: 36 >= found:35","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:49.876091017Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1280,"":"downstairs","":"downstairs","session_id":"a06228ef-19ae-435b-b1cb-98caf5aab414"} {"msg":"Next flush: 47","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:49.876100487Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1280,"":"downstairs","":"downstairs","session_id":"a06228ef-19ae-435b-b1cb-98caf5aab414"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:49.876109857Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1280,"":"downstairs","":"downstairs","session_id":"a06228ef-19ae-435b-b1cb-98caf5aab414"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:49.876120767Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1280,"session_id":"a06228ef-19ae-435b-b1cb-98caf5aab414"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:49.876129627Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1280,"client":"0","":"downstairs","session_id":"a06228ef-19ae-435b-b1cb-98caf5aab414"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:49.876139407Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1280,"client":"1","":"downstairs","session_id":"a06228ef-19ae-435b-b1cb-98caf5aab414"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:49.876149047Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1280,"client":"2","":"downstairs","session_id":"a06228ef-19ae-435b-b1cb-98caf5aab414"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:49.876163857Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1280,"session_id":"a06228ef-19ae-435b-b1cb-98caf5aab414"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:49.876173137Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1280,"session_id":"a06228ef-19ae-435b-b1cb-98caf5aab414"} {"msg":"e01fa7c1-7e63-4c69-a9ea-bfadb5ccfd64 is now active with session: a06228ef-19ae-435b-b1cb-98caf5aab414","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:49.876183137Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1280,"session_id":"a06228ef-19ae-435b-b1cb-98caf5aab414"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:49.876191927Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1280,"session_id":"a06228ef-19ae-435b-b1cb-98caf5aab414"} {"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:49.876200997Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1280} {"msg":"Activated sub_volume e01fa7c1-7e63-4c69-a9ea-bfadb5ccfd64","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:49.876209427Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1280} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:49.876217677Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1280,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:49.876465137Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1280,"task":"crutest"} 01/30 Write block 301 len 28672 data: 6 6 6 5 5 5 5 02/30 Write block 317 len 24576 data: 3 3 3 5 5 7 03/30 Write block 277 len 4096 data: 7 04/30 Write block 576 len 4096 data: 4 05/30 Flush 06/30 Write block 228 len 12288 data: 3 3 3 07/30 Write block 357 len 24576 data: 4 4 6 6 6 5 08/30 Read block 27 len 4096 09/30 Read block 343 len 28672 10/30 Read block 442 len 24576 11/30 Read block 316 len 40960 12/30 Write block 256 len 40960 data: 4 3 3 3 3 3 4 6 5 6 13/30 Read block 107 len 4096 14/30 Write block 593 len 20480 data: 2 2 3 2 2 15/30 Read block 340 len 40960 16/30 Write block 476 len 36864 data: 3 3 3 3 4 3 3 3 4 17/30 Read block 73 len 36864 18/30 Read block 182 len 8192 19/30 Read block 240 len 28672 20/30 Write block 562 len 24576 data: 2 2 2 3 3 4 21/30 Write block 66 len 36864 data: 4 3 2 2 2 2 2 2 2 22/30 Read block 285 len 16384 23/30 Read block 185 len 20480 24/30 Read block 370 len 8192 25/30 Write block 120 len 36864 data: 4 5 5 4 3 3 2 2 2 26/30 Write block 236 len 8192 data: 4 4 27/30 Read block 386 len 8192 28/30 Write block 243 len 12288 data: 5 5 5 29/30 Read block 171 len 28672 30/30 Write block 306 len 8192 data: 6 6 ---------------------------------------------------------------- Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:25 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1005 Acked Write 3 Done Done Sent false 1006 Acked Write 6 Done Done Sent false 1007 Acked Read 1 Done Done Sent false 1008 Acked Read 7 Done Done Sent false 1009 Acked Read 6 Done Done Sent false 1010 Acked Read 10 Done Done Sent false 1011 Acked Write 10 Done Done Sent false 1012 Acked Read 1 Done Done Sent false 1013 Acked Write 5 Done Done Sent false 1014 Acked Read 10 Done Done Sent false 1015 Acked Write 9 Done Done Sent false 1016 Acked Read 9 Done Done Sent false 1017 Acked Read 2 Done Done Sent false 1018 Acked Read 7 Done Done Sent false 1019 Acked Write 6 Done Done Sent false 1020 Acked Write 9 Done Done Sent false 1021 Acked Read 4 Done Done Sent false 1022 Acked Read 5 Done Done Sent false 1023 Acked Read 2 Done Done Sent false 1024 Acked Write 9 Done Done Sent false 1025 Acked Write 2 Done Done Sent false 1026 Acked Read 2 Done Done Sent false 1027 Acked Write 3 Done Done Sent false 1028 Acked Read 7 Done Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 25 28 Done 24 23 0 47 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1004)) Some(JobId(1004)) Some(JobId(1004)) Downstairs last five completed: 1004 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:49.88506207Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1280} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:49.88524106Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1280,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 32 32 32 43 43 43 F F F 1 020-039 34 34 34 46 46 46 F F F 2 040-059 33 33 33 45 45 45 F F F 3 060-079 34 34 34 46 46 46 T T F <--- 4 080-099 31 31 31 42 42 42 F F F 5 100-119 34 34 34 46 46 46 F F F 6 120-139 34 34 34 46 46 46 T T F <--- 7 140-159 34 34 34 46 46 46 F F F 8 160-179 34 34 34 46 46 46 F F F 9 180-199 34 34 34 46 46 46 F F F 10 200-219 34 34 34 46 46 46 F F F 11 220-239 31 31 31 42 42 42 T T F <--- 12 240-259 31 31 31 42 42 42 T T F <--- 13 260-279 36 36 36 47 47 47 T T F <--- 14 280-299 28 28 28 38 38 38 F F F 15 300-319 36 36 36 47 47 47 T T F <--- 16 320-339 36 36 36 47 47 47 F F F 17 340-359 32 32 32 43 43 43 T T F <--- 18 360-379 32 32 32 44 44 44 T T F <--- 19 380-399 34 34 34 46 46 46 F F F 20 400-419 33 33 33 45 45 45 F F F 21 420-439 29 29 29 39 39 39 F F F 22 440-459 34 34 34 46 46 46 F F F 23 460-479 34 34 34 46 46 46 T T F <--- 24 480-499 34 34 34 46 46 46 T T F <--- 25 500-519 30 30 30 41 41 41 F F F 26 520-539 34 34 34 46 46 46 F F F 27 540-559 34 34 34 46 46 46 F F F 28 560-579 36 36 36 47 47 47 T T F <--- 29 580-599 31 31 31 42 42 42 T T F <--- Max gen: 36, Max flush: 47 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 18 Verifying data now No change after verify Loop: 18 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 32 32 32 43 43 43 F F F 1 020-039 34 34 34 46 46 46 F F F 2 040-059 33 33 33 45 45 45 F F F 3 060-079 37 37 37 48 48 48 F F F 4 080-099 31 31 31 42 42 42 F F F 5 100-119 34 34 34 46 46 46 F F F 6 120-139 37 37 37 48 48 48 F F F 7 140-159 34 34 34 46 46 46 F F F 8 160-179 34 34 34 46 46 46 F F F 9 180-199 34 34 34 46 46 46 F F F 10 200-219 34 34 34 46 46 46 F F F 11 220-239 37 37 37 48 48 48 F F F 12 240-259 37 37 37 48 48 48 F F F 13 260-279 37 37 37 48 48 48 F F F 14 280-299 28 28 28 38 38 38 F F F 15 300-319 37 37 37 48 48 48 F F F 16 320-339 36 36 36 47 47 47 F F F 17 340-359 37 37 37 48 48 48 F F F 18 360-379 37 37 37 48 48 48 F F F 19 380-399 34 34 34 46 46 46 F F F 20 400-419 33 33 33 45 45 45 F F F 21 420-439 29 29 29 39 39 39 F F F 22 440-459 34 34 34 46 46 46 F F F 23 460-479 37 37 37 48 48 48 F F F 24 480-499 37 37 37 48 48 48 F F F 25 500-519 30 30 30 41 41 41 F F F 26 520-539 34 34 34 46 46 46 F F F 27 540-559 34 34 34 46 46 46 F F F 28 560-579 37 37 37 48 48 48 F F F 29 580-599 37 37 37 48 48 48 F F F Max gen: 37, Max flush: 48 Begin loop 19 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:54.135915575Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1288,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:54.144017077Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1288,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:54.144309368Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1288} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:54.144337178Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1288} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:54.144345908Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1288} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:54.144357778Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1288} {"msg":"Crucible 4afb65b8-4837-404a-b0bd-3848e7ab9bf8 has session id: 058f6e96-2b94-47cf-b7fa-7cfa05f8a0b8","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:54.144688398Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1288,"session_id":"058f6e96-2b94-47cf-b7fa-7cfa05f8a0b8"} {"msg":"Upstairs opts: Upstairs UUID: 4afb65b8-4837-404a-b0bd-3848e7ab9bf8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:54.144710888Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1288,"session_id":"058f6e96-2b94-47cf-b7fa-7cfa05f8a0b8"} {"msg":"Crucible stats registered with UUID: 4afb65b8-4837-404a-b0bd-3848e7ab9bf8","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:54.144721138Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1288,"session_id":"058f6e96-2b94-47cf-b7fa-7cfa05f8a0b8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:54.144730288Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1288,"session_id":"058f6e96-2b94-47cf-b7fa-7cfa05f8a0b8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:54.144882898Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1288,"":"io task","client":"2","":"downstairs","session_id":"058f6e96-2b94-47cf-b7fa-7cfa05f8a0b8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:54.144900368Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1288,"":"io task","client":"0","":"downstairs","session_id":"058f6e96-2b94-47cf-b7fa-7cfa05f8a0b8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:54.144910568Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1288,"":"io task","client":"1","":"downstairs","session_id":"058f6e96-2b94-47cf-b7fa-7cfa05f8a0b8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:54.145058179Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1288,"task":"crutest"} {"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:54.145073179Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1288} {"msg":"4afb65b8-4837-404a-b0bd-3848e7ab9bf8 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:54.145105869Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1288,"session_id":"058f6e96-2b94-47cf-b7fa-7cfa05f8a0b8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:54.145115279Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1288,"client":"0","":"downstairs","session_id":"058f6e96-2b94-47cf-b7fa-7cfa05f8a0b8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:54.145125019Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1288,"client":"1","":"downstairs","session_id":"058f6e96-2b94-47cf-b7fa-7cfa05f8a0b8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:54.145134749Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1288,"client":"2","":"downstairs","session_id":"058f6e96-2b94-47cf-b7fa-7cfa05f8a0b8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:54.145144269Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1288,"":"io task","client":"2","":"downstairs","session_id":"058f6e96-2b94-47cf-b7fa-7cfa05f8a0b8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:54.145154409Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1288,"":"io task","client":"0","":"downstairs","session_id":"058f6e96-2b94-47cf-b7fa-7cfa05f8a0b8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:54.145166479Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1288,"":"io task","client":"1","":"downstairs","session_id":"058f6e96-2b94-47cf-b7fa-7cfa05f8a0b8"} {"msg":"ds_connection connected from Ok(127.0.0.1:57069)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:54.145245879Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1288,"":"io task","client":"1","":"downstairs","session_id":"058f6e96-2b94-47cf-b7fa-7cfa05f8a0b8"} {"msg":"ds_connection connected from Ok(127.0.0.1:42958)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:54.145275889Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1288,"":"io task","client":"0","":"downstairs","session_id":"058f6e96-2b94-47cf-b7fa-7cfa05f8a0b8"} {"msg":"ds_connection connected from Ok(127.0.0.1:54350)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:54.145318219Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1288,"":"io task","client":"2","":"downstairs","session_id":"058f6e96-2b94-47cf-b7fa-7cfa05f8a0b8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:54.14593777Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1288,"client":"0","":"downstairs","session_id":"058f6e96-2b94-47cf-b7fa-7cfa05f8a0b8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:54.14596971Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1288,"client":"1","":"downstairs","session_id":"058f6e96-2b94-47cf-b7fa-7cfa05f8a0b8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:54.14598191Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1288,"client":"2","":"downstairs","session_id":"058f6e96-2b94-47cf-b7fa-7cfa05f8a0b8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:54.146292521Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1288,"session_id":"058f6e96-2b94-47cf-b7fa-7cfa05f8a0b8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:54.146302781Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1288,"session_id":"058f6e96-2b94-47cf-b7fa-7cfa05f8a0b8"} {"msg":"[0]R flush_numbers[0..12]: [43, 46, 45, 48, 42, 46, 48, 46, 46, 46, 46, 48]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:54.146327521Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1288,"":"downstairs","":"downstairs","session_id":"058f6e96-2b94-47cf-b7fa-7cfa05f8a0b8"} {"msg":"[0]R generation[0..12]: [32, 34, 33, 37, 31, 34, 37, 34, 34, 34, 34, 37]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:54.146338661Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1288,"":"downstairs","":"downstairs","session_id":"058f6e96-2b94-47cf-b7fa-7cfa05f8a0b8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:54.146349821Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1288,"":"downstairs","":"downstairs","session_id":"058f6e96-2b94-47cf-b7fa-7cfa05f8a0b8"} {"msg":"[1]R flush_numbers[0..12]: [43, 46, 45, 48, 42, 46, 48, 46, 46, 46, 46, 48]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:54.146360421Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1288,"":"downstairs","":"downstairs","session_id":"058f6e96-2b94-47cf-b7fa-7cfa05f8a0b8"} {"msg":"[1]R generation[0..12]: [32, 34, 33, 37, 31, 34, 37, 34, 34, 34, 34, 37]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:54.146370921Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1288,"":"downstairs","":"downstairs","session_id":"058f6e96-2b94-47cf-b7fa-7cfa05f8a0b8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:54.146384561Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1288,"":"downstairs","":"downstairs","session_id":"058f6e96-2b94-47cf-b7fa-7cfa05f8a0b8"} {"msg":"[2]R flush_numbers[0..12]: [43, 46, 45, 48, 42, 46, 48, 46, 46, 46, 46, 48]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:54.146395181Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1288,"":"downstairs","":"downstairs","session_id":"058f6e96-2b94-47cf-b7fa-7cfa05f8a0b8"} {"msg":"[2]R generation[0..12]: [32, 34, 33, 37, 31, 34, 37, 34, 34, 34, 34, 37]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:54.146405591Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1288,"":"downstairs","":"downstairs","session_id":"058f6e96-2b94-47cf-b7fa-7cfa05f8a0b8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:54.146416801Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1288,"":"downstairs","":"downstairs","session_id":"058f6e96-2b94-47cf-b7fa-7cfa05f8a0b8"} {"msg":"Max found gen is 38","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:54.146426301Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1288,"":"downstairs","":"downstairs","session_id":"058f6e96-2b94-47cf-b7fa-7cfa05f8a0b8"} {"msg":"Generation requested: 38 >= found:38","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:54.146436091Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1288,"":"downstairs","":"downstairs","session_id":"058f6e96-2b94-47cf-b7fa-7cfa05f8a0b8"} {"msg":"Next flush: 49","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:54.146445571Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1288,"":"downstairs","":"downstairs","session_id":"058f6e96-2b94-47cf-b7fa-7cfa05f8a0b8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:54.146455121Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1288,"":"downstairs","":"downstairs","session_id":"058f6e96-2b94-47cf-b7fa-7cfa05f8a0b8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:54.146465211Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1288,"session_id":"058f6e96-2b94-47cf-b7fa-7cfa05f8a0b8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:54.146474171Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1288,"client":"0","":"downstairs","session_id":"058f6e96-2b94-47cf-b7fa-7cfa05f8a0b8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:54.146484111Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1288,"client":"1","":"downstairs","session_id":"058f6e96-2b94-47cf-b7fa-7cfa05f8a0b8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:54.146494171Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1288,"client":"2","":"downstairs","session_id":"058f6e96-2b94-47cf-b7fa-7cfa05f8a0b8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:54.146504381Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1288,"session_id":"058f6e96-2b94-47cf-b7fa-7cfa05f8a0b8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:54.146513791Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1288,"session_id":"058f6e96-2b94-47cf-b7fa-7cfa05f8a0b8"} {"msg":"4afb65b8-4837-404a-b0bd-3848e7ab9bf8 is now active with session: 058f6e96-2b94-47cf-b7fa-7cfa05f8a0b8","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:54.146523931Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1288,"session_id":"058f6e96-2b94-47cf-b7fa-7cfa05f8a0b8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:54.146532711Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1288,"session_id":"058f6e96-2b94-47cf-b7fa-7cfa05f8a0b8"} {"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:54.146542981Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1288} {"msg":"Activated sub_volume 4afb65b8-4837-404a-b0bd-3848e7ab9bf8","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:54.146551111Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1288} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:54.146559321Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1288,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:54.146838511Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1288,"task":"crutest"} 01/30 Write block 63 len 32768 data: 6 5 5 5 4 3 3 3 02/30 Write block 178 len 16384 data: 5 4 5 5 03/30 Write block 559 len 28672 data: 4 4 3 3 3 3 4 04/30 Read block 533 len 16384 05/30 Read block 576 len 40960 06/30 Flush 07/30 Write block 64 len 36864 data: 6 6 6 5 4 4 4 3 3 08/30 Read block 450 len 40960 09/30 Read block 298 len 12288 10/30 Flush 11/30 Write block 67 len 28672 data: 6 5 5 5 4 4 3 12/30 Read block 495 len 8192 13/30 Read block 521 len 32768 14/30 Read block 154 len 12288 15/30 Read block 505 len 24576 16/30 Write block 518 len 20480 data: 3 4 5 5 5 17/30 Write block 25 len 40960 data: 6 3 3 3 3 3 3 3 3 3 18/30 Read block 253 len 20480 19/30 Write block 269 len 36864 data: 9 8 7 7 5 6 6 7 8 20/30 Read block 243 len 20480 21/30 Read block 115 len 20480 22/30 Write block 233 len 4096 data: 4 23/30 Flush 24/30 Write block 370 len 28672 data: 4 4 6 6 7 7 7 25/30 Flush 26/30 Write block 325 len 4096 data: 9 27/30 Read block 341 len 40960 28/30 Write block 555 len 40960 data: 4 4 3 4 5 5 4 4 4 4 29/30 Write block 301 len 20480 data: 7 7 7 6 6 30/30 Read block 482 len 8192 ---------------------------------------------------------------- Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:20 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1010 Acked Write 7 Done Done Done false 1011 Acked Read 2 Done Done Done false 1012 Acked Read 8 Done Done Sent false 1013 Acked Read 3 Done Done Sent false 1014 Acked Read 6 Done Done Sent false 1015 Acked Write 5 Done Done Sent false 1016 Acked Write 10 Done Done Sent false 1017 Acked Read 5 Done Done Sent false 1018 Acked Write 9 Done Done Sent false 1019 Acked Read 5 Done Done Sent false 1020 Acked Read 5 Done Done Sent false 1021 Acked Write 1 Done Done Sent false 1022 Acked Flush 0 Done Done Sent false 1023 Acked Write 7 Done Done Sent false 1024 Acked Flush 0 Done Done Sent false 1025 Acked Write 1 Done Done Sent false 1026 Acked Read 10 Done Done Sent false 1027 Acked Write 10 Done Done Sent false 1028 Acked Write 5 Done Done Sent false 1029 Acked Read 2 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 18 18 Done 20 20 2 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1009)) Downstairs last five completed: 1009 1008 1007 1006 1005 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:54.155920465Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1288} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:54.156096365Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1288,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 32 32 32 43 43 43 F F F 1 020-039 38 38 34 51 51 46 F F F <--- 2 040-059 33 33 33 45 45 45 F F F 3 060-079 38 38 38 51 51 50 F F T <--- 4 080-099 31 31 31 42 42 42 F F F 5 100-119 34 34 34 46 46 46 F F F 6 120-139 37 37 37 48 48 48 F F F 7 140-159 34 34 34 46 46 46 F F F 8 160-179 38 38 38 49 49 49 F F F 9 180-199 38 38 38 49 49 49 F F F 10 200-219 34 34 34 46 46 46 F F F 11 220-239 38 38 37 51 51 48 F F F <--- 12 240-259 37 37 37 48 48 48 F F F 13 260-279 38 38 37 51 51 48 F F F <--- 14 280-299 28 28 28 38 38 38 F F F 15 300-319 37 37 37 48 48 48 T T F <--- 16 320-339 36 36 36 47 47 47 T T F <--- 17 340-359 37 37 37 48 48 48 F F F 18 360-379 38 38 37 52 52 48 F F F <--- 19 380-399 34 34 34 46 46 46 F F F 20 400-419 33 33 33 45 45 45 F F F 21 420-439 29 29 29 39 39 39 F F F 22 440-459 34 34 34 46 46 46 F F F 23 460-479 37 37 37 48 48 48 F F F 24 480-499 37 37 37 48 48 48 F F F 25 500-519 38 38 30 51 51 41 F F F <--- 26 520-539 38 38 34 51 51 46 F F F <--- 27 540-559 38 38 38 49 49 49 T T F <--- 28 560-579 38 38 38 49 49 49 T T F <--- 29 580-599 37 37 37 48 48 48 F F F Max gen: 38, Max flush: 52 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 19 Verifying data now No change after verify Loop: 19 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 32 32 32 43 43 43 F F F 1 020-039 38 38 38 51 51 51 F F F 2 040-059 33 33 33 45 45 45 F F F 3 060-079 38 38 38 51 51 51 F F F 4 080-099 31 31 31 42 42 42 F F F 5 100-119 34 34 34 46 46 46 F F F 6 120-139 37 37 37 48 48 48 F F F 7 140-159 34 34 34 46 46 46 F F F 8 160-179 38 38 38 49 49 49 F F F 9 180-199 38 38 38 49 49 49 F F F 10 200-219 34 34 34 46 46 46 F F F 11 220-239 38 38 38 51 51 51 F F F 12 240-259 37 37 37 48 48 48 F F F 13 260-279 38 38 38 51 51 51 F F F 14 280-299 28 28 28 38 38 38 F F F 15 300-319 39 39 39 53 53 53 F F F 16 320-339 39 39 39 53 53 53 F F F 17 340-359 37 37 37 48 48 48 F F F 18 360-379 38 38 38 52 52 52 F F F 19 380-399 34 34 34 46 46 46 F F F 20 400-419 33 33 33 45 45 45 F F F 21 420-439 29 29 29 39 39 39 F F F 22 440-459 34 34 34 46 46 46 F F F 23 460-479 37 37 37 48 48 48 F F F 24 480-499 37 37 37 48 48 48 F F F 25 500-519 38 38 38 51 51 51 F F F 26 520-539 38 38 38 51 51 51 F F F 27 540-559 39 39 39 53 53 53 F F F 28 560-579 39 39 39 53 53 53 F F F 29 580-599 37 37 37 48 48 48 F F F Max gen: 39, Max flush: 53 Begin loop 20 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:58.401451122Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1296,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:58.408605803Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1296,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:58.408659643Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1296,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:58.415503683Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1296,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:58.415778293Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1296} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:58.415806423Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1296} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:58.415815013Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1296} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:58.415826823Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1296} {"msg":"Crucible 617f161d-fdd4-4df3-b95b-93d22b04f3a7 has session id: 2ac810d9-ff2c-4e61-ba92-b9ddd7fe1ffd","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:58.416227834Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1296,"session_id":"2ac810d9-ff2c-4e61-ba92-b9ddd7fe1ffd"} {"msg":"Upstairs opts: Upstairs UUID: 617f161d-fdd4-4df3-b95b-93d22b04f3a7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:58.416257534Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1296,"session_id":"2ac810d9-ff2c-4e61-ba92-b9ddd7fe1ffd"} {"msg":"Crucible stats registered with UUID: 617f161d-fdd4-4df3-b95b-93d22b04f3a7","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:58.416268654Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1296,"session_id":"2ac810d9-ff2c-4e61-ba92-b9ddd7fe1ffd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:58.416277984Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1296,"session_id":"2ac810d9-ff2c-4e61-ba92-b9ddd7fe1ffd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:58.416394504Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1296,"":"io task","client":"2","":"downstairs","session_id":"2ac810d9-ff2c-4e61-ba92-b9ddd7fe1ffd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:58.416414064Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1296,"":"io task","client":"0","":"downstairs","session_id":"2ac810d9-ff2c-4e61-ba92-b9ddd7fe1ffd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:58.416424414Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1296,"":"io task","client":"1","":"downstairs","session_id":"2ac810d9-ff2c-4e61-ba92-b9ddd7fe1ffd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:58.416590185Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1296,"task":"crutest"} {"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:58.416605345Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1296} {"msg":"617f161d-fdd4-4df3-b95b-93d22b04f3a7 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:58.416622895Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1296,"session_id":"2ac810d9-ff2c-4e61-ba92-b9ddd7fe1ffd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:58.416631785Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1296,"client":"0","":"downstairs","session_id":"2ac810d9-ff2c-4e61-ba92-b9ddd7fe1ffd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:58.416641595Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1296,"client":"1","":"downstairs","session_id":"2ac810d9-ff2c-4e61-ba92-b9ddd7fe1ffd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:58.416651235Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1296,"client":"2","":"downstairs","session_id":"2ac810d9-ff2c-4e61-ba92-b9ddd7fe1ffd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:58.416660835Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1296,"":"io task","client":"2","":"downstairs","session_id":"2ac810d9-ff2c-4e61-ba92-b9ddd7fe1ffd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:58.416670955Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1296,"":"io task","client":"0","":"downstairs","session_id":"2ac810d9-ff2c-4e61-ba92-b9ddd7fe1ffd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:58.416681025Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1296,"":"io task","client":"1","":"downstairs","session_id":"2ac810d9-ff2c-4e61-ba92-b9ddd7fe1ffd"} {"msg":"ds_connection connected from Ok(127.0.0.1:46044)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:58.416855415Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1296,"":"io task","client":"1","":"downstairs","session_id":"2ac810d9-ff2c-4e61-ba92-b9ddd7fe1ffd"} {"msg":"ds_connection connected from Ok(127.0.0.1:47304)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:58.416879915Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1296,"":"io task","client":"0","":"downstairs","session_id":"2ac810d9-ff2c-4e61-ba92-b9ddd7fe1ffd"} {"msg":"ds_connection connected from Ok(127.0.0.1:63840)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:58.416891115Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1296,"":"io task","client":"2","":"downstairs","session_id":"2ac810d9-ff2c-4e61-ba92-b9ddd7fe1ffd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:58.417289736Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1296,"client":"2","":"downstairs","session_id":"2ac810d9-ff2c-4e61-ba92-b9ddd7fe1ffd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:58.417328346Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1296,"client":"1","":"downstairs","session_id":"2ac810d9-ff2c-4e61-ba92-b9ddd7fe1ffd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:58.417410346Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1296,"client":"0","":"downstairs","session_id":"2ac810d9-ff2c-4e61-ba92-b9ddd7fe1ffd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:58.417579596Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1296,"session_id":"2ac810d9-ff2c-4e61-ba92-b9ddd7fe1ffd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:58.417595646Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1296,"session_id":"2ac810d9-ff2c-4e61-ba92-b9ddd7fe1ffd"} {"msg":"[0]R flush_numbers[0..12]: [43, 51, 45, 51, 42, 46, 48, 46, 49, 49, 46, 51]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:58.417722206Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1296,"":"downstairs","":"downstairs","session_id":"2ac810d9-ff2c-4e61-ba92-b9ddd7fe1ffd"} {"msg":"[0]R generation[0..12]: [32, 38, 33, 38, 31, 34, 37, 34, 38, 38, 34, 38]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:58.417733636Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1296,"":"downstairs","":"downstairs","session_id":"2ac810d9-ff2c-4e61-ba92-b9ddd7fe1ffd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:58.417745116Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1296,"":"downstairs","":"downstairs","session_id":"2ac810d9-ff2c-4e61-ba92-b9ddd7fe1ffd"} {"msg":"[1]R flush_numbers[0..12]: [43, 51, 45, 51, 42, 46, 48, 46, 49, 49, 46, 51]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:58.417755806Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1296,"":"downstairs","":"downstairs","session_id":"2ac810d9-ff2c-4e61-ba92-b9ddd7fe1ffd"} {"msg":"[1]R generation[0..12]: [32, 38, 33, 38, 31, 34, 37, 34, 38, 38, 34, 38]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:58.417766286Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1296,"":"downstairs","":"downstairs","session_id":"2ac810d9-ff2c-4e61-ba92-b9ddd7fe1ffd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:58.417777576Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1296,"":"downstairs","":"downstairs","session_id":"2ac810d9-ff2c-4e61-ba92-b9ddd7fe1ffd"} {"msg":"[2]R flush_numbers[0..12]: [43, 51, 45, 51, 42, 46, 48, 46, 49, 49, 46, 51]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:58.417788296Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1296,"":"downstairs","":"downstairs","session_id":"2ac810d9-ff2c-4e61-ba92-b9ddd7fe1ffd"} {"msg":"[2]R generation[0..12]: [32, 38, 33, 38, 31, 34, 37, 34, 38, 38, 34, 38]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:58.417798836Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1296,"":"downstairs","":"downstairs","session_id":"2ac810d9-ff2c-4e61-ba92-b9ddd7fe1ffd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:58.417809896Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1296,"":"downstairs","":"downstairs","session_id":"2ac810d9-ff2c-4e61-ba92-b9ddd7fe1ffd"} {"msg":"Max found gen is 40","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:58.417819406Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1296,"":"downstairs","":"downstairs","session_id":"2ac810d9-ff2c-4e61-ba92-b9ddd7fe1ffd"} {"msg":"Generation requested: 40 >= found:40","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:58.417829366Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1296,"":"downstairs","":"downstairs","session_id":"2ac810d9-ff2c-4e61-ba92-b9ddd7fe1ffd"} {"msg":"Next flush: 54","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:58.417840296Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1296,"":"downstairs","":"downstairs","session_id":"2ac810d9-ff2c-4e61-ba92-b9ddd7fe1ffd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:58.417849746Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1296,"":"downstairs","":"downstairs","session_id":"2ac810d9-ff2c-4e61-ba92-b9ddd7fe1ffd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:58.417859746Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1296,"session_id":"2ac810d9-ff2c-4e61-ba92-b9ddd7fe1ffd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:58.417868706Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1296,"client":"0","":"downstairs","session_id":"2ac810d9-ff2c-4e61-ba92-b9ddd7fe1ffd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:58.417878587Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1296,"client":"1","":"downstairs","session_id":"2ac810d9-ff2c-4e61-ba92-b9ddd7fe1ffd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:58.417888417Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1296,"client":"2","":"downstairs","session_id":"2ac810d9-ff2c-4e61-ba92-b9ddd7fe1ffd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:58.417898437Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1296,"session_id":"2ac810d9-ff2c-4e61-ba92-b9ddd7fe1ffd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:58.417907697Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1296,"session_id":"2ac810d9-ff2c-4e61-ba92-b9ddd7fe1ffd"} {"msg":"617f161d-fdd4-4df3-b95b-93d22b04f3a7 is now active with session: 2ac810d9-ff2c-4e61-ba92-b9ddd7fe1ffd","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:58.417917917Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1296,"session_id":"2ac810d9-ff2c-4e61-ba92-b9ddd7fe1ffd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:58.417926837Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1296,"session_id":"2ac810d9-ff2c-4e61-ba92-b9ddd7fe1ffd"} {"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:58.417936117Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1296} {"msg":"Activated sub_volume 617f161d-fdd4-4df3-b95b-93d22b04f3a7","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:58.417944427Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1296} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:58.417952807Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1296,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:58.418287247Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1296,"task":"crutest"} 01/30 Write block 345 len 20480 data: 3 5 5 5 4 02/30 Read block 17 len 24576 03/30 Flush 04/30 Write block 272 len 28672 data: 8 6 7 7 8 9 4 05/30 Write block 386 len 28672 data: 6 6 7 7 6 5 5 06/30 Read block 397 len 36864 07/30 Write block 576 len 4096 data: 5 08/30 Write block 397 len 24576 data: 3 3 3 2 2 2 09/30 Write block 391 len 16384 data: 6 6 3 4 10/30 Read block 516 len 16384 11/30 Read block 38 len 24576 12/30 Read block 545 len 12288 13/30 Write block 485 len 36864 data: 4 4 4 4 6 5 4 6 5 14/30 Write block 367 len 12288 data: 4 4 3 15/30 Read block 431 len 12288 16/30 Write block 444 len 20480 data: 3 3 3 4 5 17/30 Flush 18/30 Write block 158 len 28672 data: 4 4 5 5 2 2 2 19/30 Read block 486 len 36864 20/30 Flush 21/30 Write block 198 len 36864 data: 4 3 3 3 4 6 6 6 4 22/30 Write block 16 len 24576 data: 5 5 6 8 9 9 23/30 Read block 36 len 4096 24/30 Write block 330 len 28672 data: 6 5 4 2 4 5 4 25/30 Read block 85 len 32768 26/30 Write block 373 len 32768 data: 7 8 8 8 7 7 6 6 27/30 Read block 589 len 4096 28/30 Write block 210 len 24576 data: 7 7 7 5 4 5 29/30 Write block 165 len 36864 data: 2 2 2 2 3 3 3 3 3 30/30 Read block 329 len 28672 ---------------------------------------------------------------- Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1003 Acked Write 7 Done Done Done false 1004 Acked Write 7 Done Done Done false 1005 Acked Read 9 Sent Done Done false 1006 Acked Write 1 Sent Done Done false 1007 Acked Write 6 Sent Done Done false 1008 Acked Write 4 Sent Done Done false 1009 Acked Read 4 Sent Done Done false 1010 Acked Read 6 Sent Done Done false 1011 Acked Read 3 Sent Done Done false 1012 Acked Write 9 Sent Done Done false 1013 Acked Write 3 Sent Done Done false 1014 Acked Read 3 Sent Done Done false 1015 Acked Write 5 Sent Done Done false 1016 Acked Flush 0 Sent Done Done false 1017 Acked Write 7 Sent Done Done false 1018 Acked Read 9 Sent Done Done false 1019 Acked Flush 0 Sent Done Done false 1020 Acked Write 9 Sent Done Done false 1021 Acked Write 6 Sent Done Done false 1022 Acked Read 1 Sent Done Done false 1023 Acked Write 7 Sent Done Done false 1024 Acked Read 8 Sent Done Done false 1025 Acked Write 8 Sent Done Done false 1026 Acked Read 1 Sent Done Done false 1027 Acked Write 6 Sent Done Done false 1028 Acked Write 9 Sent Done Done false 1029 Acked Read 7 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 25 0 1 26 Done 2 27 26 55 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1002)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:20:58.427673801Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1296} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:20:58.427855372Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1296,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 32 32 32 43 43 43 F T T <--- 1 020-039 38 38 38 51 51 51 F T T <--- 2 040-059 33 33 33 45 45 45 F F F 3 060-079 38 38 38 51 51 51 F F F 4 080-099 31 31 31 42 42 42 F F F 5 100-119 34 34 34 46 46 46 F F F 6 120-139 37 37 37 48 48 48 F F F 7 140-159 34 40 40 46 56 56 F F F <--- 8 160-179 38 40 40 49 56 56 F T T <--- 9 180-199 38 38 38 49 49 49 F T T <--- 10 200-219 34 34 34 46 46 46 F T T <--- 11 220-239 38 38 38 51 51 51 F F F 12 240-259 37 37 37 48 48 48 F F F 13 260-279 38 40 40 51 55 55 T F F <--- 14 280-299 28 28 28 38 38 38 F F F 15 300-319 39 39 39 53 53 53 F F F 16 320-339 39 39 39 53 53 53 F T T <--- 17 340-359 40 40 40 54 54 54 F F F 18 360-379 38 40 40 52 55 55 F T T <--- 19 380-399 34 40 40 46 55 55 T T T <--- 20 400-419 33 40 40 45 55 55 F F F <--- 21 420-439 29 29 29 39 39 39 F F F 22 440-459 34 40 40 46 55 55 F F F <--- 23 460-479 37 37 37 48 48 48 F F F 24 480-499 37 40 40 48 55 55 F F F <--- 25 500-519 38 38 38 51 51 51 F F F 26 520-539 38 38 38 51 51 51 F F F 27 540-559 39 39 39 53 53 53 F F F 28 560-579 39 40 40 53 55 55 F F F <--- 29 580-599 37 37 37 48 48 48 F F F Max gen: 40, Max flush: 56 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 20 Verifying data now No change after verify Loop: 20 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 57 57 57 F F F 1 020-039 41 41 41 57 57 57 F F F 2 040-059 33 33 33 45 45 45 F F F 3 060-079 38 38 38 51 51 51 F F F 4 080-099 31 31 31 42 42 42 F F F 5 100-119 34 34 34 46 46 46 F F F 6 120-139 37 37 37 48 48 48 F F F 7 140-159 40 40 40 56 56 56 F F F 8 160-179 41 41 41 57 57 57 F F F 9 180-199 41 41 41 57 57 57 F F F 10 200-219 41 41 41 57 57 57 F F F 11 220-239 38 38 38 51 51 51 F F F 12 240-259 37 37 37 48 48 48 F F F 13 260-279 40 40 40 55 55 55 F F F 14 280-299 28 28 28 38 38 38 F F F 15 300-319 39 39 39 53 53 53 F F F 16 320-339 41 41 41 57 57 57 F F F 17 340-359 40 40 40 54 54 54 F F F 18 360-379 41 41 41 57 57 57 F F F 19 380-399 41 41 41 57 57 57 F F F 20 400-419 40 40 40 55 55 55 F F F 21 420-439 29 29 29 39 39 39 F F F 22 440-459 40 40 40 55 55 55 F F F 23 460-479 37 37 37 48 48 48 F F F 24 480-499 40 40 40 55 55 55 F F F 25 500-519 38 38 38 51 51 51 F F F 26 520-539 38 38 38 51 51 51 F F F 27 540-559 39 39 39 53 53 53 F F F 28 560-579 40 40 40 55 55 55 F F F 29 580-599 37 37 37 48 48 48 F F F Max gen: 41, Max flush: 57 Begin loop 21 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:02.689240372Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1304,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:02.697545705Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1304,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:02.697836025Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1304} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:02.697864955Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1304} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:02.697875865Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1304} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:02.697887775Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1304} {"msg":"Crucible c4769687-1a9d-46d5-bbe2-758d791d0373 has session id: 47e08c56-233f-4a8b-b810-c1aa1ae34a98","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:02.698237566Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1304,"session_id":"47e08c56-233f-4a8b-b810-c1aa1ae34a98"} {"msg":"Upstairs opts: Upstairs UUID: c4769687-1a9d-46d5-bbe2-758d791d0373, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:02.698263826Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1304,"session_id":"47e08c56-233f-4a8b-b810-c1aa1ae34a98"} {"msg":"Crucible stats registered with UUID: c4769687-1a9d-46d5-bbe2-758d791d0373","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:02.698273916Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1304,"session_id":"47e08c56-233f-4a8b-b810-c1aa1ae34a98"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:21:02.698283026Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1304,"session_id":"47e08c56-233f-4a8b-b810-c1aa1ae34a98"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:02.698430876Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1304,"":"io task","client":"0","":"downstairs","session_id":"47e08c56-233f-4a8b-b810-c1aa1ae34a98"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:02.698462186Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1304,"":"io task","client":"1","":"downstairs","session_id":"47e08c56-233f-4a8b-b810-c1aa1ae34a98"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:02.698473256Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1304,"":"io task","client":"2","":"downstairs","session_id":"47e08c56-233f-4a8b-b810-c1aa1ae34a98"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:02.698609786Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1304,"task":"crutest"} {"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:02.698626796Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1304} {"msg":"c4769687-1a9d-46d5-bbe2-758d791d0373 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:02.698642606Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1304,"session_id":"47e08c56-233f-4a8b-b810-c1aa1ae34a98"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:02.698651496Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1304,"client":"0","":"downstairs","session_id":"47e08c56-233f-4a8b-b810-c1aa1ae34a98"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:02.698661226Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1304,"client":"1","":"downstairs","session_id":"47e08c56-233f-4a8b-b810-c1aa1ae34a98"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:02.698672856Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1304,"client":"2","":"downstairs","session_id":"47e08c56-233f-4a8b-b810-c1aa1ae34a98"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:02.698682576Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1304,"":"io task","client":"2","":"downstairs","session_id":"47e08c56-233f-4a8b-b810-c1aa1ae34a98"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:02.698692716Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1304,"":"io task","client":"0","":"downstairs","session_id":"47e08c56-233f-4a8b-b810-c1aa1ae34a98"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:02.698703106Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1304,"":"io task","client":"1","":"downstairs","session_id":"47e08c56-233f-4a8b-b810-c1aa1ae34a98"} {"msg":"ds_connection connected from Ok(127.0.0.1:46052)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:02.698761096Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1304,"":"io task","client":"2","":"downstairs","session_id":"47e08c56-233f-4a8b-b810-c1aa1ae34a98"} {"msg":"ds_connection connected from Ok(127.0.0.1:64339)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:02.698786677Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1304,"":"io task","client":"0","":"downstairs","session_id":"47e08c56-233f-4a8b-b810-c1aa1ae34a98"} {"msg":"ds_connection connected from Ok(127.0.0.1:49449)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:02.698798157Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1304,"":"io task","client":"1","":"downstairs","session_id":"47e08c56-233f-4a8b-b810-c1aa1ae34a98"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:02.699476818Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1304,"client":"1","":"downstairs","session_id":"47e08c56-233f-4a8b-b810-c1aa1ae34a98"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:02.699492888Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1304,"client":"0","":"downstairs","session_id":"47e08c56-233f-4a8b-b810-c1aa1ae34a98"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:02.699638378Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1304,"client":"2","":"downstairs","session_id":"47e08c56-233f-4a8b-b810-c1aa1ae34a98"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:02.699815718Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1304,"session_id":"47e08c56-233f-4a8b-b810-c1aa1ae34a98"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:02.699833058Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1304,"session_id":"47e08c56-233f-4a8b-b810-c1aa1ae34a98"} {"msg":"[0]R flush_numbers[0..12]: [57, 57, 45, 51, 42, 46, 48, 56, 57, 57, 57, 51]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:02.699914358Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1304,"":"downstairs","":"downstairs","session_id":"47e08c56-233f-4a8b-b810-c1aa1ae34a98"} {"msg":"[0]R generation[0..12]: [41, 41, 33, 38, 31, 34, 37, 40, 41, 41, 41, 38]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:02.699925708Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1304,"":"downstairs","":"downstairs","session_id":"47e08c56-233f-4a8b-b810-c1aa1ae34a98"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:02.699936918Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1304,"":"downstairs","":"downstairs","session_id":"47e08c56-233f-4a8b-b810-c1aa1ae34a98"} {"msg":"[1]R flush_numbers[0..12]: [57, 57, 45, 51, 42, 46, 48, 56, 57, 57, 57, 51]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:02.699950218Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1304,"":"downstairs","":"downstairs","session_id":"47e08c56-233f-4a8b-b810-c1aa1ae34a98"} {"msg":"[1]R generation[0..12]: [41, 41, 33, 38, 31, 34, 37, 40, 41, 41, 41, 38]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:02.699960828Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1304,"":"downstairs","":"downstairs","session_id":"47e08c56-233f-4a8b-b810-c1aa1ae34a98"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:02.699971948Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1304,"":"downstairs","":"downstairs","session_id":"47e08c56-233f-4a8b-b810-c1aa1ae34a98"} {"msg":"[2]R flush_numbers[0..12]: [57, 57, 45, 51, 42, 46, 48, 56, 57, 57, 57, 51]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:02.699982558Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1304,"":"downstairs","":"downstairs","session_id":"47e08c56-233f-4a8b-b810-c1aa1ae34a98"} {"msg":"[2]R generation[0..12]: [41, 41, 33, 38, 31, 34, 37, 40, 41, 41, 41, 38]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:02.699992988Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1304,"":"downstairs","":"downstairs","session_id":"47e08c56-233f-4a8b-b810-c1aa1ae34a98"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:02.700004068Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1304,"":"downstairs","":"downstairs","session_id":"47e08c56-233f-4a8b-b810-c1aa1ae34a98"} {"msg":"Max found gen is 42","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:02.700013638Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1304,"":"downstairs","":"downstairs","session_id":"47e08c56-233f-4a8b-b810-c1aa1ae34a98"} {"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:02.700028668Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1304,"":"downstairs","":"downstairs","session_id":"47e08c56-233f-4a8b-b810-c1aa1ae34a98"} {"msg":"Next flush: 58","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:02.700038018Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1304,"":"downstairs","":"downstairs","session_id":"47e08c56-233f-4a8b-b810-c1aa1ae34a98"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:02.700047488Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1304,"":"downstairs","":"downstairs","session_id":"47e08c56-233f-4a8b-b810-c1aa1ae34a98"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:02.700057718Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1304,"session_id":"47e08c56-233f-4a8b-b810-c1aa1ae34a98"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:02.700066758Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1304,"client":"0","":"downstairs","session_id":"47e08c56-233f-4a8b-b810-c1aa1ae34a98"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:02.700076788Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1304,"client":"1","":"downstairs","session_id":"47e08c56-233f-4a8b-b810-c1aa1ae34a98"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:02.700086988Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1304,"client":"2","":"downstairs","session_id":"47e08c56-233f-4a8b-b810-c1aa1ae34a98"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:02.700097309Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1304,"session_id":"47e08c56-233f-4a8b-b810-c1aa1ae34a98"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:02.700109049Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1304,"session_id":"47e08c56-233f-4a8b-b810-c1aa1ae34a98"} {"msg":"c4769687-1a9d-46d5-bbe2-758d791d0373 is now active with session: 47e08c56-233f-4a8b-b810-c1aa1ae34a98","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:02.700119249Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1304,"session_id":"47e08c56-233f-4a8b-b810-c1aa1ae34a98"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:02.700128319Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1304,"session_id":"47e08c56-233f-4a8b-b810-c1aa1ae34a98"} {"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:02.700137699Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1304} {"msg":"Activated sub_volume c4769687-1a9d-46d5-bbe2-758d791d0373","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:02.700145739Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1304} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:02.700153879Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1304,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:02.700468649Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1304,"task":"crutest"} 01/30 Write block 560 len 8192 data: 6 5 02/30 Write block 316 len 28672 data: 3 4 4 4 6 6 8 03/30 Write block 341 len 20480 data: 4 3 3 3 4 04/30 Write block 360 len 4096 data: 7 05/30 Read block 223 len 12288 06/30 Read block 234 len 12288 07/30 Write block 92 len 36864 data: 4 4 4 5 4 5 5 4 3 08/30 Write block 502 len 20480 data: 3 2 2 4 3 09/30 Read block 167 len 40960 10/30 Write block 533 len 12288 data: 3 3 4 11/30 Read block 75 len 36864 12/30 Read block 84 len 40960 13/30 Write block 557 len 20480 data: 4 5 6 7 6 14/30 Read block 9 len 4096 15/30 Read block 92 len 4096 16/30 Write block 1 len 32768 data: 3 3 3 3 3 3 3 3 17/30 Write block 234 len 24576 data: 3 3 5 5 4 6 18/30 Read block 32 len 32768 19/30 Read block 101 len 12288 20/30 Read block 276 len 24576 21/30 Read block 344 len 40960 22/30 Write block 468 len 16384 data: 6 5 4 4 23/30 Read block 304 len 36864 24/30 Read block 278 len 40960 25/30 Read block 196 len 32768 26/30 Read block 80 len 32768 27/30 Write block 118 len 8192 data: 4 4 28/30 Write block 493 len 4096 data: 6 29/30 Write block 123 len 40960 data: 5 4 4 3 3 3 2 2 2 3 30/30 Write block 354 len 28672 data: 4 4 4 5 5 7 8 ---------------------------------------------------------------- Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Write 7 Done Done Done false 1002 Acked Write 5 Done Done Sent false 1003 Acked Write 1 Done Done Sent false 1004 Acked Read 3 Done Done Sent false 1005 Acked Read 3 Done Done Sent false 1006 Acked Write 9 Done Done Sent false 1007 Acked Write 5 Done Done Sent false 1008 Acked Read 10 Done Done Sent false 1009 Acked Write 3 Done Done Sent false 1010 Acked Read 9 Done Done Sent false 1011 Acked Read 10 Done Done Sent false 1012 Acked Write 5 Done Done Sent false 1013 Acked Read 1 Done Done Sent false 1014 Acked Read 1 Done Done Sent false 1015 Acked Write 8 Done Done Sent false 1016 Acked Write 6 Done Done Sent false 1017 Acked Read 8 Done Done Sent false 1018 Acked Read 3 Done Done Sent false 1019 Acked Read 6 Done Done Sent false 1020 Acked Read 10 Done Done Sent false 1021 Acked Write 4 Done Done Sent false 1022 Acked Read 9 Done Done Sent false 1023 Acked Read 10 Done Done Sent false 1024 Acked Read 8 Done Done Sent false 1025 Acked Read 8 Done Done Sent false 1026 Acked Write 2 Done Done Sent false 1027 Acked Write 1 Done Done Sent false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 28 32 Done 28 28 2 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:21:02.708322711Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1304} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:02.708507521Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1304,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 57 57 57 T T F <--- 1 020-039 41 41 41 57 57 57 F F F 2 040-059 33 33 33 45 45 45 F F F 3 060-079 38 38 38 51 51 51 F F F 4 080-099 31 31 31 42 42 42 T T F <--- 5 100-119 34 34 34 46 46 46 T T F <--- 6 120-139 37 37 37 48 48 48 T T F <--- 7 140-159 40 40 40 56 56 56 F F F 8 160-179 41 41 41 57 57 57 F F F 9 180-199 41 41 41 57 57 57 F F F 10 200-219 41 41 41 57 57 57 F F F 11 220-239 38 38 38 51 51 51 T T F <--- 12 240-259 37 37 37 48 48 48 F F F 13 260-279 40 40 40 55 55 55 F F F 14 280-299 28 28 28 38 38 38 F F F 15 300-319 39 39 39 53 53 53 T T T 16 320-339 41 41 41 57 57 57 T T T 17 340-359 40 40 40 54 54 54 T T F <--- 18 360-379 41 41 41 57 57 57 T T F <--- 19 380-399 41 41 41 57 57 57 F F F 20 400-419 40 40 40 55 55 55 F F F 21 420-439 29 29 29 39 39 39 F F F 22 440-459 40 40 40 55 55 55 F F F 23 460-479 37 37 37 48 48 48 T T F <--- 24 480-499 40 40 40 55 55 55 T T F <--- 25 500-519 38 38 38 51 51 51 T T F <--- 26 520-539 38 38 38 51 51 51 T T F <--- 27 540-559 39 39 39 53 53 53 T T F <--- 28 560-579 40 40 40 55 55 55 T T T 29 580-599 37 37 37 48 48 48 F F F Max gen: 41, Max flush: 57 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 21 Verifying data now No change after verify Loop: 21 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 58 58 58 F F F 1 020-039 41 41 41 57 57 57 F F F 2 040-059 33 33 33 45 45 45 F F F 3 060-079 38 38 38 51 51 51 F F F 4 080-099 42 42 42 58 58 58 F F F 5 100-119 42 42 42 58 58 58 F F F 6 120-139 42 42 42 58 58 58 F F F 7 140-159 40 40 40 56 56 56 F F F 8 160-179 41 41 41 57 57 57 F F F 9 180-199 41 41 41 57 57 57 F F F 10 200-219 41 41 41 57 57 57 F F F 11 220-239 42 42 42 58 58 58 F F F 12 240-259 37 37 37 48 48 48 F F F 13 260-279 40 40 40 55 55 55 F F F 14 280-299 28 28 28 38 38 38 F F F 15 300-319 42 42 42 58 58 58 F F F 16 320-339 42 42 42 58 58 58 F F F 17 340-359 42 42 42 58 58 58 F F F 18 360-379 42 42 42 58 58 58 F F F 19 380-399 41 41 41 57 57 57 F F F 20 400-419 40 40 40 55 55 55 F F F 21 420-439 29 29 29 39 39 39 F F F 22 440-459 40 40 40 55 55 55 F F F 23 460-479 42 42 42 58 58 58 F F F 24 480-499 42 42 42 58 58 58 F F F 25 500-519 42 42 42 58 58 58 F F F 26 520-539 42 42 42 58 58 58 F F F 27 540-559 42 42 42 58 58 58 F F F 28 560-579 42 42 42 58 58 58 F F F 29 580-599 37 37 37 48 48 48 F F F Max gen: 42, Max flush: 58 Begin loop 22 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:06.967361037Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1312,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:21:06.974523918Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1312,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:06.974579148Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1312,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:06.981396009Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1312,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:06.981662529Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1312} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:06.981688429Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1312} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:06.981697189Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1312} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:06.981709099Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1312} {"msg":"Crucible d544a7b7-26cd-4538-a417-4c7c5268c3d0 has session id: e47e5284-0e23-43cf-8afd-887c6b35e049","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:06.98205795Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1312,"session_id":"e47e5284-0e23-43cf-8afd-887c6b35e049"} {"msg":"Upstairs opts: Upstairs UUID: d544a7b7-26cd-4538-a417-4c7c5268c3d0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:06.98208859Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1312,"session_id":"e47e5284-0e23-43cf-8afd-887c6b35e049"} {"msg":"Crucible stats registered with UUID: d544a7b7-26cd-4538-a417-4c7c5268c3d0","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:06.98209963Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1312,"session_id":"e47e5284-0e23-43cf-8afd-887c6b35e049"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:21:06.9821088Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1312,"session_id":"e47e5284-0e23-43cf-8afd-887c6b35e049"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:06.98225104Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1312,"":"io task","client":"2","":"downstairs","session_id":"e47e5284-0e23-43cf-8afd-887c6b35e049"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:06.98228317Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1312,"":"io task","client":"0","":"downstairs","session_id":"e47e5284-0e23-43cf-8afd-887c6b35e049"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:06.98229423Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1312,"":"io task","client":"1","":"downstairs","session_id":"e47e5284-0e23-43cf-8afd-887c6b35e049"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:06.98242374Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1312,"task":"crutest"} {"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:06.98244808Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1312} {"msg":"d544a7b7-26cd-4538-a417-4c7c5268c3d0 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:06.98247193Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1312,"session_id":"e47e5284-0e23-43cf-8afd-887c6b35e049"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:06.98248142Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1312,"client":"0","":"downstairs","session_id":"e47e5284-0e23-43cf-8afd-887c6b35e049"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:06.98249124Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1312,"client":"1","":"downstairs","session_id":"e47e5284-0e23-43cf-8afd-887c6b35e049"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:06.982501Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1312,"client":"2","":"downstairs","session_id":"e47e5284-0e23-43cf-8afd-887c6b35e049"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:06.98251076Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1312,"":"io task","client":"2","":"downstairs","session_id":"e47e5284-0e23-43cf-8afd-887c6b35e049"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:06.98252101Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1312,"":"io task","client":"0","":"downstairs","session_id":"e47e5284-0e23-43cf-8afd-887c6b35e049"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:06.98253118Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1312,"":"io task","client":"1","":"downstairs","session_id":"e47e5284-0e23-43cf-8afd-887c6b35e049"} {"msg":"ds_connection connected from Ok(127.0.0.1:45669)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:06.98260339Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1312,"":"io task","client":"0","":"downstairs","session_id":"e47e5284-0e23-43cf-8afd-887c6b35e049"} {"msg":"ds_connection connected from Ok(127.0.0.1:41433)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:06.98262947Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1312,"":"io task","client":"2","":"downstairs","session_id":"e47e5284-0e23-43cf-8afd-887c6b35e049"} {"msg":"ds_connection connected from Ok(127.0.0.1:33888)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:06.982707521Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1312,"":"io task","client":"1","":"downstairs","session_id":"e47e5284-0e23-43cf-8afd-887c6b35e049"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:06.983333392Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1312,"client":"1","":"downstairs","session_id":"e47e5284-0e23-43cf-8afd-887c6b35e049"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:06.983352792Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1312,"client":"2","":"downstairs","session_id":"e47e5284-0e23-43cf-8afd-887c6b35e049"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:06.983473132Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1312,"client":"0","":"downstairs","session_id":"e47e5284-0e23-43cf-8afd-887c6b35e049"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:06.983637082Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1312,"session_id":"e47e5284-0e23-43cf-8afd-887c6b35e049"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:06.983652582Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1312,"session_id":"e47e5284-0e23-43cf-8afd-887c6b35e049"} {"msg":"[0]R flush_numbers[0..12]: [58, 57, 45, 51, 58, 58, 58, 56, 57, 57, 57, 58]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:06.983801462Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1312,"":"downstairs","":"downstairs","session_id":"e47e5284-0e23-43cf-8afd-887c6b35e049"} {"msg":"[0]R generation[0..12]: [42, 41, 33, 38, 42, 42, 42, 40, 41, 41, 41, 42]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:06.983812932Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1312,"":"downstairs","":"downstairs","session_id":"e47e5284-0e23-43cf-8afd-887c6b35e049"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:06.983824182Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1312,"":"downstairs","":"downstairs","session_id":"e47e5284-0e23-43cf-8afd-887c6b35e049"} {"msg":"[1]R flush_numbers[0..12]: [58, 57, 45, 51, 58, 58, 58, 56, 57, 57, 57, 58]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:06.983834862Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1312,"":"downstairs","":"downstairs","session_id":"e47e5284-0e23-43cf-8afd-887c6b35e049"} {"msg":"[1]R generation[0..12]: [42, 41, 33, 38, 42, 42, 42, 40, 41, 41, 41, 42]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:06.983845382Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1312,"":"downstairs","":"downstairs","session_id":"e47e5284-0e23-43cf-8afd-887c6b35e049"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:06.983856512Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1312,"":"downstairs","":"downstairs","session_id":"e47e5284-0e23-43cf-8afd-887c6b35e049"} {"msg":"[2]R flush_numbers[0..12]: [58, 57, 45, 51, 58, 58, 58, 56, 57, 57, 57, 58]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:06.983869302Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1312,"":"downstairs","":"downstairs","session_id":"e47e5284-0e23-43cf-8afd-887c6b35e049"} {"msg":"[2]R generation[0..12]: [42, 41, 33, 38, 42, 42, 42, 40, 41, 41, 41, 42]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:06.983880012Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1312,"":"downstairs","":"downstairs","session_id":"e47e5284-0e23-43cf-8afd-887c6b35e049"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:06.983891092Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1312,"":"downstairs","":"downstairs","session_id":"e47e5284-0e23-43cf-8afd-887c6b35e049"} {"msg":"Max found gen is 43","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:06.983900532Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1312,"":"downstairs","":"downstairs","session_id":"e47e5284-0e23-43cf-8afd-887c6b35e049"} {"msg":"Generation requested: 44 >= found:43","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:06.983910202Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1312,"":"downstairs","":"downstairs","session_id":"e47e5284-0e23-43cf-8afd-887c6b35e049"} {"msg":"Next flush: 59","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:06.983919492Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1312,"":"downstairs","":"downstairs","session_id":"e47e5284-0e23-43cf-8afd-887c6b35e049"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:06.983928982Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1312,"":"downstairs","":"downstairs","session_id":"e47e5284-0e23-43cf-8afd-887c6b35e049"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:06.983938812Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1312,"session_id":"e47e5284-0e23-43cf-8afd-887c6b35e049"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:06.983947652Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1312,"client":"0","":"downstairs","session_id":"e47e5284-0e23-43cf-8afd-887c6b35e049"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:06.983957433Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1312,"client":"1","":"downstairs","session_id":"e47e5284-0e23-43cf-8afd-887c6b35e049"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:06.983967093Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1312,"client":"2","":"downstairs","session_id":"e47e5284-0e23-43cf-8afd-887c6b35e049"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:06.983982633Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1312,"session_id":"e47e5284-0e23-43cf-8afd-887c6b35e049"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:06.983992043Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1312,"session_id":"e47e5284-0e23-43cf-8afd-887c6b35e049"} {"msg":"d544a7b7-26cd-4538-a417-4c7c5268c3d0 is now active with session: e47e5284-0e23-43cf-8afd-887c6b35e049","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:06.984002293Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1312,"session_id":"e47e5284-0e23-43cf-8afd-887c6b35e049"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:06.984011153Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1312,"session_id":"e47e5284-0e23-43cf-8afd-887c6b35e049"} {"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:06.984020443Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1312} {"msg":"Activated sub_volume d544a7b7-26cd-4538-a417-4c7c5268c3d0","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:06.984028763Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1312} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:06.984038433Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1312,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:06.984346853Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1312,"task":"crutest"} 01/30 Write block 458 len 4096 data: 6 02/30 Read block 162 len 12288 03/30 Read block 553 len 24576 04/30 Write block 275 len 4096 data: 8 05/30 Read block 110 len 8192 06/30 Flush 07/30 Write block 197 len 28672 data: 5 5 4 4 4 5 7 08/30 Write block 256 len 20480 data: 5 4 4 4 4 09/30 Write block 391 len 40960 data: 7 7 4 5 4 3 4 4 4 3 10/30 Write block 104 len 28672 data: 4 5 5 4 4 5 6 11/30 Read block 267 len 32768 12/30 Flush 13/30 Write block 358 len 36864 data: 6 8 9 7 6 5 5 5 5 14/30 Read block 177 len 20480 15/30 Flush 16/30 Flush 17/30 Write block 483 len 20480 data: 4 5 5 5 5 18/30 Write block 509 len 20480 data: 2 2 2 3 3 19/30 Write block 232 len 36864 data: 3 5 4 4 6 6 5 7 6 20/30 Read block 490 len 24576 21/30 Read block 302 len 32768 22/30 Read block 556 len 36864 23/30 Write block 474 len 32768 data: 3 3 4 4 4 4 5 4 24/30 Write block 468 len 32768 data: 7 6 5 5 3 3 4 4 25/30 Write block 362 len 20480 data: 7 6 6 6 6 26/30 Read block 545 len 4096 27/30 Read block 220 len 24576 28/30 Read block 406 len 8192 29/30 Read block 450 len 4096 30/30 Read block 462 len 24576 ---------------------------------------------------------------- Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 7 Done Done Done false 1007 Acked Write 5 Done Done Done false 1008 Acked Write 10 Sent Done Done false 1009 Acked Write 7 Sent Done Done false 1010 Acked Read 8 Sent Done Done false 1011 Acked Flush 0 Sent Done Done false 1012 Acked Write 9 Sent Done Done false 1013 Acked Read 5 Sent Done Done false 1014 Acked Flush 0 Sent Done Done false 1015 Acked Flush 0 Sent Done Done false 1016 Acked Write 5 Sent Done Done false 1017 Acked Write 5 Sent Done Done false 1018 Acked Write 9 Sent Done Done false 1019 Acked Read 6 Sent Done Done false 1020 Acked Read 8 Sent Done Done false 1021 Acked Read 9 Sent Done Done false 1022 Acked Write 8 Sent Done Done false 1023 Acked Write 8 Sent Done Done false 1024 Acked Write 5 Sent Done Done false 1025 Acked Read 1 Sent Done Done false 1026 Acked Read 6 Sent Done Done false 1027 Acked Read 2 Sent Done Done false 1028 Acked Read 1 Sent Done Done false 1029 Acked Read 6 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 22 0 0 22 Done 2 24 24 50 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1005)) Some(JobId(1015)) Some(JobId(1015)) Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:21:06.993021106Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1312} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:06.993199406Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1312,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 58 58 58 F F F 1 020-039 41 41 41 57 57 57 F F F 2 040-059 33 33 33 45 45 45 F F F 3 060-079 38 38 38 51 51 51 F F F 4 080-099 42 42 42 58 58 58 F F F 5 100-119 42 44 44 58 60 60 F F F <--- 6 120-139 42 42 42 58 58 58 F F F 7 140-159 40 40 40 56 56 56 F F F 8 160-179 41 41 41 57 57 57 F F F 9 180-199 41 44 44 57 60 60 T F F <--- 10 200-219 41 44 44 57 60 60 T F F <--- 11 220-239 42 42 42 58 58 58 F T T <--- 12 240-259 37 44 44 48 60 60 T T T <--- 13 260-279 44 44 44 59 60 60 T F F <--- 14 280-299 28 28 28 38 38 38 F F F 15 300-319 42 42 42 58 58 58 F F F 16 320-339 42 42 42 58 58 58 F F F 17 340-359 42 44 44 58 61 61 F F F <--- 18 360-379 42 44 44 58 61 61 F T T <--- 19 380-399 41 44 44 57 60 60 F F F <--- 20 400-419 40 44 44 55 60 60 F F F <--- 21 420-439 29 29 29 39 39 39 F F F 22 440-459 44 44 44 59 59 59 F F F 23 460-479 42 42 42 58 58 58 F T T <--- 24 480-499 42 42 42 58 58 58 F T T <--- 25 500-519 42 42 42 58 58 58 F T T <--- 26 520-539 42 42 42 58 58 58 F F F 27 540-559 42 42 42 58 58 58 F F F 28 560-579 42 42 42 58 58 58 F F F 29 580-599 37 37 37 48 48 48 F F F Max gen: 44, Max flush: 61 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 22 Verifying data now No change after verify Loop: 22 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 58 58 58 F F F 1 020-039 41 41 41 57 57 57 F F F 2 040-059 33 33 33 45 45 45 F F F 3 060-079 38 38 38 51 51 51 F F F 4 080-099 42 42 42 58 58 58 F F F 5 100-119 44 44 44 60 60 60 F F F 6 120-139 42 42 42 58 58 58 F F F 7 140-159 40 40 40 56 56 56 F F F 8 160-179 41 41 41 57 57 57 F F F 9 180-199 44 44 44 60 60 60 F F F 10 200-219 44 44 44 60 60 60 F F F 11 220-239 45 45 45 62 62 62 F F F 12 240-259 45 45 45 62 62 62 F F F 13 260-279 44 44 44 60 60 60 F F F 14 280-299 28 28 28 38 38 38 F F F 15 300-319 42 42 42 58 58 58 F F F 16 320-339 42 42 42 58 58 58 F F F 17 340-359 44 44 44 61 61 61 F F F 18 360-379 45 45 45 62 62 62 F F F 19 380-399 44 44 44 60 60 60 F F F 20 400-419 44 44 44 60 60 60 F F F 21 420-439 29 29 29 39 39 39 F F F 22 440-459 44 44 44 59 59 59 F F F 23 460-479 45 45 45 62 62 62 F F F 24 480-499 45 45 45 62 62 62 F F F 25 500-519 45 45 45 62 62 62 F F F 26 520-539 42 42 42 58 58 58 F F F 27 540-559 42 42 42 58 58 58 F F F 28 560-579 42 42 42 58 58 58 F F F 29 580-599 37 37 37 48 48 48 F F F Max gen: 45, Max flush: 62 Begin loop 23 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:11.256039679Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1320,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:11.264293062Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1320,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:11.264590332Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1320} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:11.264618972Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1320} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:11.264627652Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1320} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:11.264680392Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1320} {"msg":"Crucible c16a141a-4ca7-46df-985f-cb57abe1ee4d has session id: 3dede8dc-d0be-4460-b89b-e85ca5e20e36","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:11.264974463Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1320,"session_id":"3dede8dc-d0be-4460-b89b-e85ca5e20e36"} {"msg":"Upstairs opts: Upstairs UUID: c16a141a-4ca7-46df-985f-cb57abe1ee4d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:11.264994843Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1320,"session_id":"3dede8dc-d0be-4460-b89b-e85ca5e20e36"} {"msg":"Crucible stats registered with UUID: c16a141a-4ca7-46df-985f-cb57abe1ee4d","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:11.265005403Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1320,"session_id":"3dede8dc-d0be-4460-b89b-e85ca5e20e36"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:21:11.265014923Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1320,"session_id":"3dede8dc-d0be-4460-b89b-e85ca5e20e36"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:11.265159213Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1320,"":"io task","client":"2","":"downstairs","session_id":"3dede8dc-d0be-4460-b89b-e85ca5e20e36"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:11.265177263Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1320,"":"io task","client":"0","":"downstairs","session_id":"3dede8dc-d0be-4460-b89b-e85ca5e20e36"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:11.265187603Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1320,"":"io task","client":"1","":"downstairs","session_id":"3dede8dc-d0be-4460-b89b-e85ca5e20e36"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:11.265336193Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1320,"task":"crutest"} {"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:11.265351803Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1320} {"msg":"c16a141a-4ca7-46df-985f-cb57abe1ee4d active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:11.265389953Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1320,"session_id":"3dede8dc-d0be-4460-b89b-e85ca5e20e36"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:11.265399473Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1320,"client":"0","":"downstairs","session_id":"3dede8dc-d0be-4460-b89b-e85ca5e20e36"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:11.265409293Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1320,"client":"1","":"downstairs","session_id":"3dede8dc-d0be-4460-b89b-e85ca5e20e36"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:11.265418993Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1320,"client":"2","":"downstairs","session_id":"3dede8dc-d0be-4460-b89b-e85ca5e20e36"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:11.265428743Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1320,"":"io task","client":"2","":"downstairs","session_id":"3dede8dc-d0be-4460-b89b-e85ca5e20e36"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:11.265448473Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1320,"":"io task","client":"0","":"downstairs","session_id":"3dede8dc-d0be-4460-b89b-e85ca5e20e36"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:11.265458463Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1320,"":"io task","client":"1","":"downstairs","session_id":"3dede8dc-d0be-4460-b89b-e85ca5e20e36"} {"msg":"ds_connection connected from Ok(127.0.0.1:54168)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:11.265512074Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1320,"":"io task","client":"2","":"downstairs","session_id":"3dede8dc-d0be-4460-b89b-e85ca5e20e36"} {"msg":"ds_connection connected from Ok(127.0.0.1:48084)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:11.265537494Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1320,"":"io task","client":"0","":"downstairs","session_id":"3dede8dc-d0be-4460-b89b-e85ca5e20e36"} {"msg":"ds_connection connected from Ok(127.0.0.1:63947)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:11.265548914Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1320,"":"io task","client":"1","":"downstairs","session_id":"3dede8dc-d0be-4460-b89b-e85ca5e20e36"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:11.266189705Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1320,"client":"1","":"downstairs","session_id":"3dede8dc-d0be-4460-b89b-e85ca5e20e36"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:11.266203215Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1320,"client":"0","":"downstairs","session_id":"3dede8dc-d0be-4460-b89b-e85ca5e20e36"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:11.266223185Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1320,"client":"2","":"downstairs","session_id":"3dede8dc-d0be-4460-b89b-e85ca5e20e36"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:11.266520655Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1320,"session_id":"3dede8dc-d0be-4460-b89b-e85ca5e20e36"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:11.266531655Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1320,"session_id":"3dede8dc-d0be-4460-b89b-e85ca5e20e36"} {"msg":"[0]R flush_numbers[0..12]: [58, 57, 45, 51, 58, 60, 58, 56, 57, 60, 60, 62]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:11.266568985Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1320,"":"downstairs","":"downstairs","session_id":"3dede8dc-d0be-4460-b89b-e85ca5e20e36"} {"msg":"[0]R generation[0..12]: [42, 41, 33, 38, 42, 44, 42, 40, 41, 44, 44, 45]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:11.266580155Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1320,"":"downstairs","":"downstairs","session_id":"3dede8dc-d0be-4460-b89b-e85ca5e20e36"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:11.266591455Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1320,"":"downstairs","":"downstairs","session_id":"3dede8dc-d0be-4460-b89b-e85ca5e20e36"} {"msg":"[1]R flush_numbers[0..12]: [58, 57, 45, 51, 58, 60, 58, 56, 57, 60, 60, 62]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:11.266602045Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1320,"":"downstairs","":"downstairs","session_id":"3dede8dc-d0be-4460-b89b-e85ca5e20e36"} {"msg":"[1]R generation[0..12]: [42, 41, 33, 38, 42, 44, 42, 40, 41, 44, 44, 45]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:11.266612525Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1320,"":"downstairs","":"downstairs","session_id":"3dede8dc-d0be-4460-b89b-e85ca5e20e36"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:11.266623505Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1320,"":"downstairs","":"downstairs","session_id":"3dede8dc-d0be-4460-b89b-e85ca5e20e36"} {"msg":"[2]R flush_numbers[0..12]: [58, 57, 45, 51, 58, 60, 58, 56, 57, 60, 60, 62]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:11.266633995Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1320,"":"downstairs","":"downstairs","session_id":"3dede8dc-d0be-4460-b89b-e85ca5e20e36"} {"msg":"[2]R generation[0..12]: [42, 41, 33, 38, 42, 44, 42, 40, 41, 44, 44, 45]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:11.266644495Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1320,"":"downstairs","":"downstairs","session_id":"3dede8dc-d0be-4460-b89b-e85ca5e20e36"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:11.266655315Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1320,"":"downstairs","":"downstairs","session_id":"3dede8dc-d0be-4460-b89b-e85ca5e20e36"} {"msg":"Max found gen is 46","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:11.266664785Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1320,"":"downstairs","":"downstairs","session_id":"3dede8dc-d0be-4460-b89b-e85ca5e20e36"} {"msg":"Generation requested: 46 >= found:46","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:11.266674425Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1320,"":"downstairs","":"downstairs","session_id":"3dede8dc-d0be-4460-b89b-e85ca5e20e36"} {"msg":"Next flush: 63","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:11.266683765Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1320,"":"downstairs","":"downstairs","session_id":"3dede8dc-d0be-4460-b89b-e85ca5e20e36"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:11.266693185Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1320,"":"downstairs","":"downstairs","session_id":"3dede8dc-d0be-4460-b89b-e85ca5e20e36"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:11.266703125Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1320,"session_id":"3dede8dc-d0be-4460-b89b-e85ca5e20e36"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:11.266713485Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1320,"client":"0","":"downstairs","session_id":"3dede8dc-d0be-4460-b89b-e85ca5e20e36"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:11.266723155Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1320,"client":"1","":"downstairs","session_id":"3dede8dc-d0be-4460-b89b-e85ca5e20e36"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:11.266732745Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1320,"client":"2","":"downstairs","session_id":"3dede8dc-d0be-4460-b89b-e85ca5e20e36"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:11.266742575Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1320,"session_id":"3dede8dc-d0be-4460-b89b-e85ca5e20e36"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:11.266751765Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1320,"session_id":"3dede8dc-d0be-4460-b89b-e85ca5e20e36"} {"msg":"c16a141a-4ca7-46df-985f-cb57abe1ee4d is now active with session: 3dede8dc-d0be-4460-b89b-e85ca5e20e36","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:11.266761775Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1320,"session_id":"3dede8dc-d0be-4460-b89b-e85ca5e20e36"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:11.266770525Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1320,"session_id":"3dede8dc-d0be-4460-b89b-e85ca5e20e36"} {"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:11.266779695Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1320} {"msg":"Activated sub_volume c16a141a-4ca7-46df-985f-cb57abe1ee4d","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:11.266788155Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1320} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:11.266796635Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1320,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:11.267089756Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1320,"task":"crutest"} 01/30 Write block 333 len 36864 data: 3 5 6 5 5 3 4 5 5 02/30 Read block 457 len 8192 03/30 Write block 276 len 4096 data: 9 04/30 Write block 360 len 4096 data: 10 05/30 Read block 541 len 28672 06/30 Write block 211 len 4096 data: 8 07/30 Read block 375 len 12288 08/30 Write block 270 len 8192 data: 9 8 09/30 Read block 149 len 24576 10/30 Read block 573 len 4096 11/30 Read block 303 len 16384 12/30 Write block 78 len 28672 data: 4 5 5 4 5 5 4 13/30 Flush 14/30 Write block 92 len 24576 data: 5 5 5 6 5 6 15/30 Read block 230 len 24576 16/30 Write block 278 len 4096 data: 5 17/30 Read block 458 len 32768 18/30 Read block 235 len 20480 19/30 Read block 462 len 36864 20/30 Write block 139 len 36864 data: 9 8 8 5 4 5 4 4 5 21/30 Write block 267 len 12288 data: 7 8 10 22/30 Write block 341 len 32768 data: 6 4 4 4 5 6 6 6 23/30 Write block 85 len 28672 data: 5 6 5 4 4 5 4 24/30 Write block 496 len 16384 data: 3 2 2 2 25/30 Write block 525 len 36864 data: 6 5 7 7 6 4 4 4 4 26/30 Write block 571 len 4096 data: 5 27/30 Write block 26 len 24576 data: 4 4 4 4 4 4 28/30 Write block 266 len 28672 data: 7 8 9 11 10 9 9 29/30 Write block 309 len 24576 data: 4 4 3 3 3 3 30/30 Read block 327 len 28672 ---------------------------------------------------------------- Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Read 2 Done Done Done false 1002 Acked Write 1 Done Done Done false 1003 Acked Write 1 Done Done Done false 1004 Acked Read 7 Done Done Done false 1005 Acked Write 1 Done Done Sent false 1006 Acked Read 3 Done Done Sent false 1007 Acked Write 2 Done Done Sent false 1008 Acked Read 6 Done Done Sent false 1009 Acked Read 1 Done Done Sent false 1010 Acked Read 4 Done Done Sent false 1011 Acked Write 7 Done Done Sent false 1012 Acked Flush 0 Done Done Sent false 1013 Acked Write 6 Done Done Sent false 1014 Acked Read 6 Done Done Sent false 1015 Acked Write 1 Done Done Sent false 1016 Acked Read 8 Done Done Sent false 1017 Acked Read 5 Done Done Sent false 1018 Acked Read 9 Done Done Sent false 1019 Acked Write 9 Done Done Sent false 1020 Acked Write 3 Done Done Sent false 1021 Acked Write 8 Done Done Sent false 1022 Acked Write 7 Done Done Sent false 1023 Acked Write 4 Done Done Sent false 1024 Acked Write 9 Done Done Sent false 1025 Acked Write 1 Done Done Sent false 1026 Acked Write 6 Done Done Sent false 1027 Acked Write 7 Done Done Sent false 1028 Acked Write 6 Done Done Sent false 1029 Acked Read 7 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 25 26 Done 29 30 5 64 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1012)) Some(JobId(1012)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:21:11.274811648Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1320} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:11.274990318Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1320,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 58 58 58 F F F 1 020-039 41 41 41 57 57 57 T T F <--- 2 040-059 33 33 33 45 45 45 F F F 3 060-079 46 46 38 63 63 51 F F F <--- 4 080-099 46 46 42 63 63 58 T T F <--- 5 100-119 44 44 44 60 60 60 F F F 6 120-139 42 42 42 58 58 58 T T F <--- 7 140-159 40 40 40 56 56 56 T T F <--- 8 160-179 41 41 41 57 57 57 F F F 9 180-199 44 44 44 60 60 60 F F F 10 200-219 46 46 44 63 63 60 F F F <--- 11 220-239 45 45 45 62 62 62 F F F 12 240-259 45 45 45 62 62 62 F F F 13 260-279 46 46 44 63 63 60 T T T <--- 14 280-299 28 28 28 38 38 38 F F F 15 300-319 42 42 42 58 58 58 T T F <--- 16 320-339 46 46 42 63 63 58 F F T <--- 17 340-359 46 46 44 63 63 61 T T T <--- 18 360-379 46 46 45 63 63 62 F F T <--- 19 380-399 44 44 44 60 60 60 F F F 20 400-419 44 44 44 60 60 60 F F F 21 420-439 29 29 29 39 39 39 F F F 22 440-459 44 44 44 59 59 59 F F F 23 460-479 45 45 45 62 62 62 F F F 24 480-499 45 45 45 62 62 62 T T F <--- 25 500-519 45 45 45 62 62 62 F F F 26 520-539 42 42 42 58 58 58 T T F <--- 27 540-559 42 42 42 58 58 58 F F F 28 560-579 42 42 42 58 58 58 T T F <--- 29 580-599 37 37 37 48 48 48 F F F Max gen: 46, Max flush: 63 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 23 Verifying data now No change after verify Loop: 23 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 58 58 58 F F F 1 020-039 47 47 47 64 64 64 F F F 2 040-059 33 33 33 45 45 45 F F F 3 060-079 46 46 46 63 63 63 F F F 4 080-099 47 47 47 64 64 64 F F F 5 100-119 44 44 44 60 60 60 F F F 6 120-139 47 47 47 64 64 64 F F F 7 140-159 47 47 47 64 64 64 F F F 8 160-179 41 41 41 57 57 57 F F F 9 180-199 44 44 44 60 60 60 F F F 10 200-219 46 46 46 63 63 63 F F F 11 220-239 45 45 45 62 62 62 F F F 12 240-259 45 45 45 62 62 62 F F F 13 260-279 47 47 47 64 64 64 F F F 14 280-299 28 28 28 38 38 38 F F F 15 300-319 47 47 47 64 64 64 F F F 16 320-339 46 46 46 63 63 63 F F F 17 340-359 47 47 47 64 64 64 F F F 18 360-379 46 46 46 63 63 63 F F F 19 380-399 44 44 44 60 60 60 F F F 20 400-419 44 44 44 60 60 60 F F F 21 420-439 29 29 29 39 39 39 F F F 22 440-459 44 44 44 59 59 59 F F F 23 460-479 45 45 45 62 62 62 F F F 24 480-499 47 47 47 64 64 64 F F F 25 500-519 45 45 45 62 62 62 F F F 26 520-539 47 47 47 64 64 64 F F F 27 540-559 42 42 42 58 58 58 F F F 28 560-579 47 47 47 64 64 64 F F F 29 580-599 37 37 37 48 48 48 F F F Max gen: 47, Max flush: 64 Begin loop 24 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:15.535100116Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1328,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:15.543252089Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1328,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:15.543580119Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1328} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:15.543614569Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1328} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:15.543623999Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1328} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:15.543638489Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1328} {"msg":"Crucible fe6e1509-ffe1-4ee8-8eef-81d309099309 has session id: ffb80e51-9ce1-48cf-91e8-d951626f838e","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:15.54397494Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1328,"session_id":"ffb80e51-9ce1-48cf-91e8-d951626f838e"} {"msg":"Upstairs opts: Upstairs UUID: fe6e1509-ffe1-4ee8-8eef-81d309099309, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:15.54401043Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1328,"session_id":"ffb80e51-9ce1-48cf-91e8-d951626f838e"} {"msg":"Crucible stats registered with UUID: fe6e1509-ffe1-4ee8-8eef-81d309099309","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:15.54402059Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1328,"session_id":"ffb80e51-9ce1-48cf-91e8-d951626f838e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:21:15.54402973Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1328,"session_id":"ffb80e51-9ce1-48cf-91e8-d951626f838e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:15.54416314Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1328,"":"io task","client":"1","":"downstairs","session_id":"ffb80e51-9ce1-48cf-91e8-d951626f838e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:15.54418152Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1328,"":"io task","client":"0","":"downstairs","session_id":"ffb80e51-9ce1-48cf-91e8-d951626f838e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:15.54419157Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1328,"":"io task","client":"2","":"downstairs","session_id":"ffb80e51-9ce1-48cf-91e8-d951626f838e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:15.54433928Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1328,"task":"crutest"} {"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:15.54436316Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1328} {"msg":"fe6e1509-ffe1-4ee8-8eef-81d309099309 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:15.54437951Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1328,"session_id":"ffb80e51-9ce1-48cf-91e8-d951626f838e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:15.54438857Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1328,"client":"0","":"downstairs","session_id":"ffb80e51-9ce1-48cf-91e8-d951626f838e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:15.54439835Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1328,"client":"1","":"downstairs","session_id":"ffb80e51-9ce1-48cf-91e8-d951626f838e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:15.54440804Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1328,"client":"2","":"downstairs","session_id":"ffb80e51-9ce1-48cf-91e8-d951626f838e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:15.544417621Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1328,"":"io task","client":"2","":"downstairs","session_id":"ffb80e51-9ce1-48cf-91e8-d951626f838e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:15.544429821Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1328,"":"io task","client":"0","":"downstairs","session_id":"ffb80e51-9ce1-48cf-91e8-d951626f838e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:15.544439871Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1328,"":"io task","client":"1","":"downstairs","session_id":"ffb80e51-9ce1-48cf-91e8-d951626f838e"} {"msg":"ds_connection connected from Ok(127.0.0.1:40067)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:15.544507731Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1328,"":"io task","client":"1","":"downstairs","session_id":"ffb80e51-9ce1-48cf-91e8-d951626f838e"} {"msg":"ds_connection connected from Ok(127.0.0.1:52991)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:15.544533861Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1328,"":"io task","client":"0","":"downstairs","session_id":"ffb80e51-9ce1-48cf-91e8-d951626f838e"} {"msg":"ds_connection connected from Ok(127.0.0.1:47711)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:15.544545421Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1328,"":"io task","client":"2","":"downstairs","session_id":"ffb80e51-9ce1-48cf-91e8-d951626f838e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:15.545178112Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1328,"client":"0","":"downstairs","session_id":"ffb80e51-9ce1-48cf-91e8-d951626f838e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:15.545245422Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1328,"client":"2","":"downstairs","session_id":"ffb80e51-9ce1-48cf-91e8-d951626f838e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:15.545332762Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1328,"client":"1","":"downstairs","session_id":"ffb80e51-9ce1-48cf-91e8-d951626f838e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:15.545459702Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1328,"session_id":"ffb80e51-9ce1-48cf-91e8-d951626f838e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:15.545491272Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1328,"session_id":"ffb80e51-9ce1-48cf-91e8-d951626f838e"} {"msg":"[0]R flush_numbers[0..12]: [58, 64, 45, 63, 64, 60, 64, 64, 57, 60, 63, 62]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:15.545594702Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1328,"":"downstairs","":"downstairs","session_id":"ffb80e51-9ce1-48cf-91e8-d951626f838e"} {"msg":"[0]R generation[0..12]: [42, 47, 33, 46, 47, 44, 47, 47, 41, 44, 46, 45]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:15.545606172Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1328,"":"downstairs","":"downstairs","session_id":"ffb80e51-9ce1-48cf-91e8-d951626f838e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:15.545617692Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1328,"":"downstairs","":"downstairs","session_id":"ffb80e51-9ce1-48cf-91e8-d951626f838e"} {"msg":"[1]R flush_numbers[0..12]: [58, 64, 45, 63, 64, 60, 64, 64, 57, 60, 63, 62]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:15.545628372Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1328,"":"downstairs","":"downstairs","session_id":"ffb80e51-9ce1-48cf-91e8-d951626f838e"} {"msg":"[1]R generation[0..12]: [42, 47, 33, 46, 47, 44, 47, 47, 41, 44, 46, 45]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:15.545638772Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1328,"":"downstairs","":"downstairs","session_id":"ffb80e51-9ce1-48cf-91e8-d951626f838e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:15.545652252Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1328,"":"downstairs","":"downstairs","session_id":"ffb80e51-9ce1-48cf-91e8-d951626f838e"} {"msg":"[2]R flush_numbers[0..12]: [58, 64, 45, 63, 64, 60, 64, 64, 57, 60, 63, 62]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:15.545662772Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1328,"":"downstairs","":"downstairs","session_id":"ffb80e51-9ce1-48cf-91e8-d951626f838e"} {"msg":"[2]R generation[0..12]: [42, 47, 33, 46, 47, 44, 47, 47, 41, 44, 46, 45]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:15.545673112Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1328,"":"downstairs","":"downstairs","session_id":"ffb80e51-9ce1-48cf-91e8-d951626f838e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:15.545684232Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1328,"":"downstairs","":"downstairs","session_id":"ffb80e51-9ce1-48cf-91e8-d951626f838e"} {"msg":"Max found gen is 48","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:15.545693602Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1328,"":"downstairs","":"downstairs","session_id":"ffb80e51-9ce1-48cf-91e8-d951626f838e"} {"msg":"Generation requested: 48 >= found:48","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:15.545703222Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1328,"":"downstairs","":"downstairs","session_id":"ffb80e51-9ce1-48cf-91e8-d951626f838e"} {"msg":"Next flush: 65","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:15.545712682Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1328,"":"downstairs","":"downstairs","session_id":"ffb80e51-9ce1-48cf-91e8-d951626f838e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:15.545722232Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1328,"":"downstairs","":"downstairs","session_id":"ffb80e51-9ce1-48cf-91e8-d951626f838e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:15.545732132Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1328,"session_id":"ffb80e51-9ce1-48cf-91e8-d951626f838e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:15.545741123Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1328,"client":"0","":"downstairs","session_id":"ffb80e51-9ce1-48cf-91e8-d951626f838e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:15.545750743Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1328,"client":"1","":"downstairs","session_id":"ffb80e51-9ce1-48cf-91e8-d951626f838e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:15.545760593Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1328,"client":"2","":"downstairs","session_id":"ffb80e51-9ce1-48cf-91e8-d951626f838e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:15.545770673Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1328,"session_id":"ffb80e51-9ce1-48cf-91e8-d951626f838e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:15.545779943Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1328,"session_id":"ffb80e51-9ce1-48cf-91e8-d951626f838e"} {"msg":"fe6e1509-ffe1-4ee8-8eef-81d309099309 is now active with session: ffb80e51-9ce1-48cf-91e8-d951626f838e","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:15.545790003Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1328,"session_id":"ffb80e51-9ce1-48cf-91e8-d951626f838e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:15.545800383Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1328,"session_id":"ffb80e51-9ce1-48cf-91e8-d951626f838e"} {"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:15.545809533Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1328} {"msg":"Activated sub_volume fe6e1509-ffe1-4ee8-8eef-81d309099309","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:15.545817783Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1328} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:15.545826043Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1328,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:15.546128043Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1328,"task":"crutest"} 01/30 Write block 388 len 12288 data: 8 8 7 02/30 Read block 132 len 36864 03/30 Flush 04/30 Write block 21 len 36864 data: 10 8 8 7 7 5 5 5 5 05/30 Flush 06/30 Write block 528 len 12288 data: 8 7 5 07/30 Read block 294 len 28672 08/30 Read block 383 len 28672 09/30 Read block 341 len 20480 10/30 Write block 315 len 24576 data: 2 4 5 5 5 7 11/30 Write block 293 len 28672 data: 6 5 5 5 4 4 5 12/30 Read block 417 len 12288 13/30 Read block 388 len 28672 14/30 Read block 355 len 36864 15/30 Write block 30 len 4096 data: 5 16/30 Write block 245 len 32768 data: 6 5 5 6 4 3 3 2 17/30 Write block 525 len 20480 data: 7 6 8 9 8 18/30 Read block 192 len 12288 19/30 Read block 382 len 12288 20/30 Read block 181 len 24576 21/30 Flush 22/30 Write block 52 len 40960 data: 7 6 4 4 4 4 5 4 5 6 23/30 Write block 10 len 20480 data: 2 2 2 2 3 24/30 Read block 437 len 28672 25/30 Read block 133 len 4096 26/30 Read block 53 len 20480 27/30 Write block 404 len 28672 data: 2 2 2 2 3 2 2 28/30 Write block 255 len 4096 data: 6 29/30 Read block 439 len 24576 30/30 Write block 328 len 4096 data: 8 ---------------------------------------------------------------- Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Sent Done false 1001 Acked Read 9 Done Sent Done false 1002 Acked Flush 0 Done Sent Done false 1003 Acked Write 9 Done Sent Done false 1004 Acked Flush 0 Done Sent Done false 1005 Acked Write 3 Done Sent Done false 1006 Acked Read 7 Done Sent Done false 1007 Acked Read 7 Done Sent Done false 1008 Acked Read 5 Done Sent Done false 1009 Acked Write 6 Done Sent Done false 1010 Acked Write 7 Done Sent Done false 1011 Acked Read 3 Done Sent Done false 1012 Acked Read 7 Done Sent Done false 1013 Acked Read 9 Done Sent Done false 1014 Acked Write 1 Done Sent Done false 1015 Acked Write 8 Done Sent Done false 1016 Acked Write 5 Done Sent Done false 1017 Acked Read 3 Done Sent Done false 1018 Acked Read 3 Done Sent Done false 1019 Acked Read 6 Done Sent Done false 1020 Acked Flush 0 Done Sent Done false 1021 Acked Write 10 Done Sent Done false 1022 Acked Write 5 Done Sent Done false 1023 Acked Read 7 Done Sent Done false 1024 Acked Read 1 Done Sent Done false 1025 Acked Read 5 Done Sent Done false 1026 Acked Write 7 Done Sent Done false 1027 Acked Write 1 Done Sent Done false 1028 Acked Read 6 Done Sent Done false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 30 1 32 Done 29 0 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) None Some(JobId(1020)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:21:15.555073417Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1328} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:15.555259547Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1328,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 58 58 58 T F T <--- 1 020-039 48 47 48 67 64 67 F F F <--- 2 040-059 33 33 33 45 45 45 T F T <--- 3 060-079 46 46 46 63 63 63 T F T <--- 4 080-099 47 47 47 64 64 64 F F F 5 100-119 44 44 44 60 60 60 F F F 6 120-139 47 47 47 64 64 64 F F F 7 140-159 47 47 47 64 64 64 F F F 8 160-179 41 41 41 57 57 57 F F F 9 180-199 44 44 44 60 60 60 F F F 10 200-219 46 46 46 63 63 63 F F F 11 220-239 45 45 45 62 62 62 F F F 12 240-259 48 45 48 67 62 67 T F T <--- 13 260-279 47 47 47 64 64 64 F F F 14 280-299 48 28 48 67 38 67 F F F <--- 15 300-319 48 47 48 67 64 67 F F F <--- 16 320-339 48 46 48 67 63 67 T F T <--- 17 340-359 47 47 47 64 64 64 F F F 18 360-379 46 46 46 63 63 63 F F F 19 380-399 48 44 48 65 60 65 F F F <--- 20 400-419 44 44 44 60 60 60 T F T <--- 21 420-439 29 29 29 39 39 39 F F F 22 440-459 44 44 44 59 59 59 F F F 23 460-479 45 45 45 62 62 62 F F F 24 480-499 47 47 47 64 64 64 F F F 25 500-519 45 45 45 62 62 62 F F F 26 520-539 48 47 48 67 64 67 F F F <--- 27 540-559 42 42 42 58 58 58 F F F 28 560-579 47 47 47 64 64 64 F F F 29 580-599 37 37 37 48 48 48 F F F Max gen: 48, Max flush: 67 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 24 Verifying data now No change after verify Loop: 24 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 68 68 68 F F F 1 020-039 48 48 48 67 67 67 F F F 2 040-059 49 49 49 68 68 68 F F F 3 060-079 49 49 49 68 68 68 F F F 4 080-099 47 47 47 64 64 64 F F F 5 100-119 44 44 44 60 60 60 F F F 6 120-139 47 47 47 64 64 64 F F F 7 140-159 47 47 47 64 64 64 F F F 8 160-179 41 41 41 57 57 57 F F F 9 180-199 44 44 44 60 60 60 F F F 10 200-219 46 46 46 63 63 63 F F F 11 220-239 45 45 45 62 62 62 F F F 12 240-259 49 49 49 68 68 68 F F F 13 260-279 47 47 47 64 64 64 F F F 14 280-299 48 48 48 67 67 67 F F F 15 300-319 48 48 48 67 67 67 F F F 16 320-339 49 49 49 68 68 68 F F F 17 340-359 47 47 47 64 64 64 F F F 18 360-379 46 46 46 63 63 63 F F F 19 380-399 48 48 48 65 65 65 F F F 20 400-419 49 49 49 68 68 68 F F F 21 420-439 29 29 29 39 39 39 F F F 22 440-459 44 44 44 59 59 59 F F F 23 460-479 45 45 45 62 62 62 F F F 24 480-499 47 47 47 64 64 64 F F F 25 500-519 45 45 45 62 62 62 F F F 26 520-539 48 48 48 67 67 67 F F F 27 540-559 42 42 42 58 58 58 F F F 28 560-579 47 47 47 64 64 64 F F F 29 580-599 37 37 37 48 48 48 F F F Max gen: 49, Max flush: 68 Begin loop 25 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:19.800293253Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1336,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:21:19.807368523Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1336,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:19.807419374Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1336,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:19.814253204Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1336,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:19.814517614Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1336} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:19.814548314Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1336} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:19.814556904Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1336} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:19.814568694Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1336} {"msg":"Crucible 7eec4827-9a48-4e67-ac9b-808e83ee8275 has session id: b052b938-752e-429d-be23-42f26f8fb40e","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:19.814916105Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1336,"session_id":"b052b938-752e-429d-be23-42f26f8fb40e"} {"msg":"Upstairs opts: Upstairs UUID: 7eec4827-9a48-4e67-ac9b-808e83ee8275, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:19.814943855Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1336,"session_id":"b052b938-752e-429d-be23-42f26f8fb40e"} {"msg":"Crucible stats registered with UUID: 7eec4827-9a48-4e67-ac9b-808e83ee8275","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:19.814957445Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1336,"session_id":"b052b938-752e-429d-be23-42f26f8fb40e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:21:19.814966845Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1336,"session_id":"b052b938-752e-429d-be23-42f26f8fb40e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:19.815094445Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1336,"":"io task","client":"1","":"downstairs","session_id":"b052b938-752e-429d-be23-42f26f8fb40e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:19.815113575Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1336,"":"io task","client":"2","":"downstairs","session_id":"b052b938-752e-429d-be23-42f26f8fb40e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:19.815123765Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1336,"":"io task","client":"0","":"downstairs","session_id":"b052b938-752e-429d-be23-42f26f8fb40e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:19.815274405Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1336,"task":"crutest"} {"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:19.815298465Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1336} {"msg":"7eec4827-9a48-4e67-ac9b-808e83ee8275 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:19.815314675Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1336,"session_id":"b052b938-752e-429d-be23-42f26f8fb40e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:19.815323675Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1336,"client":"0","":"downstairs","session_id":"b052b938-752e-429d-be23-42f26f8fb40e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:19.815333445Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1336,"client":"1","":"downstairs","session_id":"b052b938-752e-429d-be23-42f26f8fb40e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:19.815343215Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1336,"client":"2","":"downstairs","session_id":"b052b938-752e-429d-be23-42f26f8fb40e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:19.815352975Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1336,"":"io task","client":"2","":"downstairs","session_id":"b052b938-752e-429d-be23-42f26f8fb40e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:19.815363055Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1336,"":"io task","client":"0","":"downstairs","session_id":"b052b938-752e-429d-be23-42f26f8fb40e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:19.815373026Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1336,"":"io task","client":"1","":"downstairs","session_id":"b052b938-752e-429d-be23-42f26f8fb40e"} {"msg":"ds_connection connected from Ok(127.0.0.1:50174)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:19.815414546Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1336,"":"io task","client":"2","":"downstairs","session_id":"b052b938-752e-429d-be23-42f26f8fb40e"} {"msg":"ds_connection connected from Ok(127.0.0.1:42507)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:19.815462906Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1336,"":"io task","client":"1","":"downstairs","session_id":"b052b938-752e-429d-be23-42f26f8fb40e"} {"msg":"ds_connection connected from Ok(127.0.0.1:58989)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:19.815573896Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1336,"":"io task","client":"0","":"downstairs","session_id":"b052b938-752e-429d-be23-42f26f8fb40e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:19.816123417Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1336,"client":"2","":"downstairs","session_id":"b052b938-752e-429d-be23-42f26f8fb40e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:19.816143797Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1336,"client":"0","":"downstairs","session_id":"b052b938-752e-429d-be23-42f26f8fb40e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:19.816171657Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1336,"client":"1","":"downstairs","session_id":"b052b938-752e-429d-be23-42f26f8fb40e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:19.816448607Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1336,"session_id":"b052b938-752e-429d-be23-42f26f8fb40e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:19.816459817Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1336,"session_id":"b052b938-752e-429d-be23-42f26f8fb40e"} {"msg":"[0]R flush_numbers[0..12]: [68, 67, 68, 68, 64, 60, 64, 64, 57, 60, 63, 62]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:19.816478367Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1336,"":"downstairs","":"downstairs","session_id":"b052b938-752e-429d-be23-42f26f8fb40e"} {"msg":"[0]R generation[0..12]: [49, 48, 49, 49, 47, 44, 47, 47, 41, 44, 46, 45]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:19.816489107Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1336,"":"downstairs","":"downstairs","session_id":"b052b938-752e-429d-be23-42f26f8fb40e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:19.816500327Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1336,"":"downstairs","":"downstairs","session_id":"b052b938-752e-429d-be23-42f26f8fb40e"} {"msg":"[1]R flush_numbers[0..12]: [68, 67, 68, 68, 64, 60, 64, 64, 57, 60, 63, 62]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:19.816511037Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1336,"":"downstairs","":"downstairs","session_id":"b052b938-752e-429d-be23-42f26f8fb40e"} {"msg":"[1]R generation[0..12]: [49, 48, 49, 49, 47, 44, 47, 47, 41, 44, 46, 45]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:19.816521697Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1336,"":"downstairs","":"downstairs","session_id":"b052b938-752e-429d-be23-42f26f8fb40e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:19.816532827Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1336,"":"downstairs","":"downstairs","session_id":"b052b938-752e-429d-be23-42f26f8fb40e"} {"msg":"[2]R flush_numbers[0..12]: [68, 67, 68, 68, 64, 60, 64, 64, 57, 60, 63, 62]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:19.816543627Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1336,"":"downstairs","":"downstairs","session_id":"b052b938-752e-429d-be23-42f26f8fb40e"} {"msg":"[2]R generation[0..12]: [49, 48, 49, 49, 47, 44, 47, 47, 41, 44, 46, 45]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:19.816554077Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1336,"":"downstairs","":"downstairs","session_id":"b052b938-752e-429d-be23-42f26f8fb40e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:19.816565267Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1336,"":"downstairs","":"downstairs","session_id":"b052b938-752e-429d-be23-42f26f8fb40e"} {"msg":"Max found gen is 50","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:19.816576857Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1336,"":"downstairs","":"downstairs","session_id":"b052b938-752e-429d-be23-42f26f8fb40e"} {"msg":"Generation requested: 50 >= found:50","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:19.816586757Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1336,"":"downstairs","":"downstairs","session_id":"b052b938-752e-429d-be23-42f26f8fb40e"} {"msg":"Next flush: 69","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:19.816596127Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1336,"":"downstairs","":"downstairs","session_id":"b052b938-752e-429d-be23-42f26f8fb40e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:19.816605607Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1336,"":"downstairs","":"downstairs","session_id":"b052b938-752e-429d-be23-42f26f8fb40e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:19.816615387Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1336,"session_id":"b052b938-752e-429d-be23-42f26f8fb40e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:19.816624257Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1336,"client":"0","":"downstairs","session_id":"b052b938-752e-429d-be23-42f26f8fb40e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:19.816633977Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1336,"client":"1","":"downstairs","session_id":"b052b938-752e-429d-be23-42f26f8fb40e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:19.816643597Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1336,"client":"2","":"downstairs","session_id":"b052b938-752e-429d-be23-42f26f8fb40e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:19.816653507Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1336,"session_id":"b052b938-752e-429d-be23-42f26f8fb40e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:19.816662697Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1336,"session_id":"b052b938-752e-429d-be23-42f26f8fb40e"} {"msg":"7eec4827-9a48-4e67-ac9b-808e83ee8275 is now active with session: b052b938-752e-429d-be23-42f26f8fb40e","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:19.816672777Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1336,"session_id":"b052b938-752e-429d-be23-42f26f8fb40e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:19.816681637Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1336,"session_id":"b052b938-752e-429d-be23-42f26f8fb40e"} {"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:19.816690828Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1336} {"msg":"Activated sub_volume 7eec4827-9a48-4e67-ac9b-808e83ee8275","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:19.816699048Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1336} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:19.816707208Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1336,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:19.816998108Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1336,"task":"crutest"} 01/30 Write block 290 len 40960 data: 5 3 4 7 6 6 6 5 5 6 02/30 Read block 484 len 36864 03/30 Write block 501 len 24576 data: 2 4 3 3 5 4 04/30 Write block 237 len 12288 data: 7 6 8 05/30 Write block 298 len 28672 data: 6 7 5 8 8 8 7 06/30 Write block 228 len 4096 data: 4 07/30 Read block 277 len 24576 08/30 Write block 542 len 32768 data: 5 5 5 4 4 3 4 4 09/30 Read block 367 len 36864 10/30 Read block 303 len 8192 11/30 Write block 149 len 40960 data: 7 6 6 6 5 7 8 6 4 5 12/30 Read block 448 len 40960 13/30 Write block 178 len 4096 data: 6 14/30 Write block 575 len 16384 data: 4 6 5 5 15/30 Read block 194 len 4096 16/30 Write block 257 len 4096 data: 5 17/30 Read block 524 len 12288 18/30 Read block 218 len 4096 19/30 Read block 143 len 24576 20/30 Read block 590 len 20480 21/30 Write block 31 len 20480 data: 5 4 4 4 3 22/30 Read block 481 len 28672 23/30 Write block 269 len 36864 data: 12 11 10 10 7 8 9 10 10 24/30 Write block 73 len 24576 data: 4 3 2 3 4 5 25/30 Read block 401 len 4096 26/30 Flush 27/30 Write block 594 len 20480 data: 3 4 3 3 2 28/30 Read block 504 len 36864 29/30 Write block 250 len 24576 data: 4 4 3 3 4 7 30/30 Read block 415 len 24576 ---------------------------------------------------------------- Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Read 9 Done Done Done false 1002 Acked Write 6 Sent Done Done false 1003 Acked Write 3 Sent Done Done false 1004 Acked Write 7 Sent Done Done false 1005 Acked Write 1 Sent Done Done false 1006 Acked Read 6 Sent Done Done false 1007 Acked Write 8 Sent Done Done false 1008 Acked Read 9 Sent Done Done false 1009 Acked Read 2 Sent Done Done false 1010 Acked Write 10 Sent Done Done false 1011 Acked Read 10 Sent Done Done false 1012 Acked Write 1 Sent Done Done false 1013 Acked Write 4 Sent Done Done false 1014 Acked Read 1 Sent Done Done false 1015 Acked Write 1 Sent Done Done false 1016 Acked Read 3 Sent Done Done false 1017 Acked Read 1 Sent Done Done false 1018 Acked Read 6 Sent Done Done false 1019 Acked Read 5 Sent Done Done false 1020 Acked Write 5 Sent Done Done false 1021 Acked Read 7 Sent Done Done false 1022 Acked Write 9 Sent Done Done false 1023 Acked Write 6 Sent Done Done false 1024 Acked Read 1 Sent Done Done false 1025 Acked Flush 0 Sent Done Done false 1026 Acked Write 5 Sent Done Done false 1027 Acked Read 9 Sent Done Done false 1028 Acked Write 6 Sent Done Done false 1029 Acked Read 6 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 1 0 29 Done 2 29 30 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:21:19.826082451Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1336} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:19.826266271Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1336,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 68 68 68 F F F 1 020-039 48 50 50 67 69 69 F F F <--- 2 040-059 49 49 49 68 68 68 F F F 3 060-079 49 50 50 68 69 69 F F F <--- 4 080-099 47 47 47 64 64 64 F F F 5 100-119 44 44 44 60 60 60 F F F 6 120-139 47 47 47 64 64 64 F F F 7 140-159 47 50 50 64 69 69 F F F <--- 8 160-179 41 50 50 57 69 69 F F F <--- 9 180-199 44 44 44 60 60 60 F F F 10 200-219 46 46 46 63 63 63 F F F 11 220-239 45 50 50 62 69 69 F F F <--- 12 240-259 49 50 50 68 69 69 F T T <--- 13 260-279 47 50 50 64 69 69 F F F <--- 14 280-299 48 50 50 67 69 69 T F F <--- 15 300-319 48 50 50 67 69 69 F F F <--- 16 320-339 49 49 49 68 68 68 F F F 17 340-359 47 47 47 64 64 64 F F F 18 360-379 46 46 46 63 63 63 F F F 19 380-399 48 48 48 65 65 65 F F F 20 400-419 49 49 49 68 68 68 F F F 21 420-439 29 29 29 39 39 39 F F F 22 440-459 44 44 44 59 59 59 F F F 23 460-479 45 45 45 62 62 62 F F F 24 480-499 47 47 47 64 64 64 F F F 25 500-519 45 50 50 62 69 69 F F F <--- 26 520-539 48 48 48 67 67 67 F F F 27 540-559 42 50 50 58 69 69 F F F <--- 28 560-579 47 50 50 64 69 69 F F F <--- 29 580-599 37 37 37 48 48 48 F T T <--- Max gen: 50, Max flush: 69 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 25 Verifying data now No change after verify Loop: 25 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 68 68 68 F F F 1 020-039 50 50 50 69 69 69 F F F 2 040-059 49 49 49 68 68 68 F F F 3 060-079 50 50 50 69 69 69 F F F 4 080-099 47 47 47 64 64 64 F F F 5 100-119 44 44 44 60 60 60 F F F 6 120-139 47 47 47 64 64 64 F F F 7 140-159 50 50 50 69 69 69 F F F 8 160-179 50 50 50 69 69 69 F F F 9 180-199 44 44 44 60 60 60 F F F 10 200-219 46 46 46 63 63 63 F F F 11 220-239 50 50 50 69 69 69 F F F 12 240-259 51 51 51 70 70 70 F F F 13 260-279 50 50 50 69 69 69 F F F 14 280-299 50 50 50 69 69 69 F F F 15 300-319 50 50 50 69 69 69 F F F 16 320-339 49 49 49 68 68 68 F F F 17 340-359 47 47 47 64 64 64 F F F 18 360-379 46 46 46 63 63 63 F F F 19 380-399 48 48 48 65 65 65 F F F 20 400-419 49 49 49 68 68 68 F F F 21 420-439 29 29 29 39 39 39 F F F 22 440-459 44 44 44 59 59 59 F F F 23 460-479 45 45 45 62 62 62 F F F 24 480-499 47 47 47 64 64 64 F F F 25 500-519 50 50 50 69 69 69 F F F 26 520-539 48 48 48 67 67 67 F F F 27 540-559 50 50 50 69 69 69 F F F 28 560-579 50 50 50 69 69 69 F F F 29 580-599 51 51 51 70 70 70 F F F Max gen: 51, Max flush: 70 Begin loop 26 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:24.077586897Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1344,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:24.085897309Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1344,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:24.08619093Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1344} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:24.08622145Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1344} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:24.08623003Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1344} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:24.08624221Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1344} {"msg":"Crucible 7183e3b6-043b-46ab-8edf-4bac47209fb4 has session id: 4cb8bff1-a756-4699-9812-89f642415f32","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:24.08658106Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1344,"session_id":"4cb8bff1-a756-4699-9812-89f642415f32"} {"msg":"Upstairs opts: Upstairs UUID: 7183e3b6-043b-46ab-8edf-4bac47209fb4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:24.08660101Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1344,"session_id":"4cb8bff1-a756-4699-9812-89f642415f32"} {"msg":"Crucible stats registered with UUID: 7183e3b6-043b-46ab-8edf-4bac47209fb4","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:24.08661127Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1344,"session_id":"4cb8bff1-a756-4699-9812-89f642415f32"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:21:24.0866206Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1344,"session_id":"4cb8bff1-a756-4699-9812-89f642415f32"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:24.086775651Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1344,"":"io task","client":"2","":"downstairs","session_id":"4cb8bff1-a756-4699-9812-89f642415f32"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:24.086807181Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1344,"":"io task","client":"0","":"downstairs","session_id":"4cb8bff1-a756-4699-9812-89f642415f32"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:24.086818331Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1344,"":"io task","client":"1","":"downstairs","session_id":"4cb8bff1-a756-4699-9812-89f642415f32"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:24.086951271Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1344,"task":"crutest"} {"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:24.086966681Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1344} {"msg":"7183e3b6-043b-46ab-8edf-4bac47209fb4 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:24.086987171Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1344,"session_id":"4cb8bff1-a756-4699-9812-89f642415f32"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:24.086996351Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1344,"client":"0","":"downstairs","session_id":"4cb8bff1-a756-4699-9812-89f642415f32"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:24.087006181Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1344,"client":"1","":"downstairs","session_id":"4cb8bff1-a756-4699-9812-89f642415f32"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:24.087016101Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1344,"client":"2","":"downstairs","session_id":"4cb8bff1-a756-4699-9812-89f642415f32"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:24.087025771Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1344,"":"io task","client":"2","":"downstairs","session_id":"4cb8bff1-a756-4699-9812-89f642415f32"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:24.087035901Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1344,"":"io task","client":"0","":"downstairs","session_id":"4cb8bff1-a756-4699-9812-89f642415f32"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:24.087046011Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1344,"":"io task","client":"1","":"downstairs","session_id":"4cb8bff1-a756-4699-9812-89f642415f32"} {"msg":"ds_connection connected from Ok(127.0.0.1:49653)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:24.087164211Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1344,"":"io task","client":"2","":"downstairs","session_id":"4cb8bff1-a756-4699-9812-89f642415f32"} {"msg":"ds_connection connected from Ok(127.0.0.1:36772)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:24.087175451Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1344,"":"io task","client":"1","":"downstairs","session_id":"4cb8bff1-a756-4699-9812-89f642415f32"} {"msg":"ds_connection connected from Ok(127.0.0.1:42997)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:24.087185951Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1344,"":"io task","client":"0","":"downstairs","session_id":"4cb8bff1-a756-4699-9812-89f642415f32"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:24.087926082Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1344,"client":"2","":"downstairs","session_id":"4cb8bff1-a756-4699-9812-89f642415f32"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:24.087967723Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1344,"client":"1","":"downstairs","session_id":"4cb8bff1-a756-4699-9812-89f642415f32"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:24.088029193Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1344,"client":"0","":"downstairs","session_id":"4cb8bff1-a756-4699-9812-89f642415f32"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:24.088272123Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1344,"session_id":"4cb8bff1-a756-4699-9812-89f642415f32"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:24.088296633Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1344,"session_id":"4cb8bff1-a756-4699-9812-89f642415f32"} {"msg":"[0]R flush_numbers[0..12]: [68, 69, 68, 69, 64, 60, 64, 69, 69, 60, 63, 69]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:24.088354233Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1344,"":"downstairs","":"downstairs","session_id":"4cb8bff1-a756-4699-9812-89f642415f32"} {"msg":"[0]R generation[0..12]: [49, 50, 49, 50, 47, 44, 47, 50, 50, 44, 46, 50]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:24.088366213Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1344,"":"downstairs","":"downstairs","session_id":"4cb8bff1-a756-4699-9812-89f642415f32"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:24.088377723Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1344,"":"downstairs","":"downstairs","session_id":"4cb8bff1-a756-4699-9812-89f642415f32"} {"msg":"[1]R flush_numbers[0..12]: [68, 69, 68, 69, 64, 60, 64, 69, 69, 60, 63, 69]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:24.088388443Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1344,"":"downstairs","":"downstairs","session_id":"4cb8bff1-a756-4699-9812-89f642415f32"} {"msg":"[1]R generation[0..12]: [49, 50, 49, 50, 47, 44, 47, 50, 50, 44, 46, 50]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:24.088399063Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1344,"":"downstairs","":"downstairs","session_id":"4cb8bff1-a756-4699-9812-89f642415f32"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:24.088410163Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1344,"":"downstairs","":"downstairs","session_id":"4cb8bff1-a756-4699-9812-89f642415f32"} {"msg":"[2]R flush_numbers[0..12]: [68, 69, 68, 69, 64, 60, 64, 69, 69, 60, 63, 69]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:24.088420853Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1344,"":"downstairs","":"downstairs","session_id":"4cb8bff1-a756-4699-9812-89f642415f32"} {"msg":"[2]R generation[0..12]: [49, 50, 49, 50, 47, 44, 47, 50, 50, 44, 46, 50]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:24.088431273Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1344,"":"downstairs","":"downstairs","session_id":"4cb8bff1-a756-4699-9812-89f642415f32"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:24.088442393Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1344,"":"downstairs","":"downstairs","session_id":"4cb8bff1-a756-4699-9812-89f642415f32"} {"msg":"Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:24.088451943Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1344,"":"downstairs","":"downstairs","session_id":"4cb8bff1-a756-4699-9812-89f642415f32"} {"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:24.088461863Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1344,"":"downstairs","":"downstairs","session_id":"4cb8bff1-a756-4699-9812-89f642415f32"} {"msg":"Next flush: 71","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:24.088471433Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1344,"":"downstairs","":"downstairs","session_id":"4cb8bff1-a756-4699-9812-89f642415f32"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:24.088481053Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1344,"":"downstairs","":"downstairs","session_id":"4cb8bff1-a756-4699-9812-89f642415f32"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:24.088490953Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1344,"session_id":"4cb8bff1-a756-4699-9812-89f642415f32"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:24.088499843Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1344,"client":"0","":"downstairs","session_id":"4cb8bff1-a756-4699-9812-89f642415f32"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:24.088511293Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1344,"client":"1","":"downstairs","session_id":"4cb8bff1-a756-4699-9812-89f642415f32"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:24.088521193Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1344,"client":"2","":"downstairs","session_id":"4cb8bff1-a756-4699-9812-89f642415f32"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:24.088531313Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1344,"session_id":"4cb8bff1-a756-4699-9812-89f642415f32"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:24.088540593Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1344,"session_id":"4cb8bff1-a756-4699-9812-89f642415f32"} {"msg":"7183e3b6-043b-46ab-8edf-4bac47209fb4 is now active with session: 4cb8bff1-a756-4699-9812-89f642415f32","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:24.088550703Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1344,"session_id":"4cb8bff1-a756-4699-9812-89f642415f32"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:24.088559613Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1344,"session_id":"4cb8bff1-a756-4699-9812-89f642415f32"} {"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:24.088568873Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1344} {"msg":"Activated sub_volume 7183e3b6-043b-46ab-8edf-4bac47209fb4","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:24.088577223Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1344} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:24.088585673Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1344,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:24.088862914Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1344,"task":"crutest"} 01/30 Write block 510 len 4096 data: 3 02/30 Write block 404 len 28672 data: 3 3 3 3 4 3 3 03/30 Write block 38 len 20480 data: 3 4 3 4 3 04/30 Read block 148 len 4096 05/30 Flush 06/30 Write block 237 len 28672 data: 8 7 9 7 6 6 6 07/30 Read block 86 len 12288 08/30 Flush 09/30 Write block 393 len 28672 data: 5 6 5 4 5 5 5 10/30 Read block 89 len 16384 11/30 Read block 179 len 28672 12/30 Read block 484 len 36864 13/30 Write block 421 len 28672 data: 4 5 5 5 4 3 3 14/30 Flush 15/30 Write block 35 len 36864 data: 4 4 3 4 5 4 5 4 3 16/30 Read block 83 len 4096 17/30 Write block 497 len 24576 data: 3 3 3 2 3 5 18/30 Read block 138 len 4096 19/30 Read block 531 len 16384 20/30 Read block 158 len 32768 21/30 Write block 428 len 40960 data: 3 3 2 3 2 2 2 2 2 2 22/30 Write block 349 len 24576 data: 5 3 3 3 4 5 23/30 Read block 442 len 4096 24/30 Read block 266 len 4096 25/30 Read block 126 len 4096 26/30 Read block 448 len 8192 27/30 Write block 245 len 20480 data: 7 6 6 7 5 28/30 Read block 365 len 40960 29/30 Read block 78 len 20480 30/30 Read block 223 len 8192 ---------------------------------------------------------------- Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Write 7 Done Done Done false 1002 Acked Write 5 Done Done Done false 1003 Acked Read 1 Done Done Done false 1004 Acked Flush 0 Done Sent Done false 1005 Acked Write 7 Done Sent Done false 1006 Acked Read 3 Done Sent Done false 1007 Acked Flush 0 Done Sent Done false 1008 Acked Write 7 Done Sent Done false 1009 Acked Read 4 Done Sent Done false 1010 Acked Read 7 Done Sent Done false 1011 Acked Read 9 Done Sent Done false 1012 Acked Write 7 Done Sent Done false 1013 Acked Flush 0 Done Sent Done false 1014 Acked Write 9 Done Sent Done false 1015 Acked Read 1 Done Sent Done false 1016 Acked Write 6 Done Sent Done false 1017 Acked Read 1 Done Sent Done false 1018 Acked Read 4 Done Sent Done false 1019 Acked Read 8 Done Sent Done false 1020 Acked Write 10 Done Sent Done false 1021 Acked Write 6 Done Sent Done false 1022 Acked Read 1 Done Sent Done false 1023 Acked Read 1 Done Sent Done false 1024 Acked Read 1 Done Sent Done false 1025 Acked Read 2 Done Sent Done false 1026 Acked Write 5 Done Sent Done false 1027 Acked Read 10 Done Sent Done false 1028 Acked Read 5 Done Sent Done false 1029 Acked Read 2 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 26 0 26 Done 30 4 30 64 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1013)) None Some(JobId(1013)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:21:24.098259378Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1344} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:24.098426558Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1344,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 68 68 68 F F F 1 020-039 52 50 52 71 69 71 T T T <--- 2 040-059 52 49 52 71 68 71 T T T <--- 3 060-079 50 50 50 69 69 69 F F F 4 080-099 47 47 47 64 64 64 F F F 5 100-119 44 44 44 60 60 60 F F F 6 120-139 47 47 47 64 64 64 F F F 7 140-159 50 50 50 69 69 69 F F F 8 160-179 50 50 50 69 69 69 F F F 9 180-199 44 44 44 60 60 60 F F F 10 200-219 46 46 46 63 63 63 F F F 11 220-239 52 50 52 72 69 72 F F F <--- 12 240-259 52 51 52 72 70 72 T F T <--- 13 260-279 50 50 50 69 69 69 F F F 14 280-299 50 50 50 69 69 69 F F F 15 300-319 50 50 50 69 69 69 F F F 16 320-339 49 49 49 68 68 68 F F F 17 340-359 47 47 47 64 64 64 T F T <--- 18 360-379 46 46 46 63 63 63 F F F 19 380-399 52 48 52 73 65 73 F F F <--- 20 400-419 52 49 52 71 68 71 F T F <--- 21 420-439 52 29 52 73 39 73 T F T <--- 22 440-459 44 44 44 59 59 59 F F F 23 460-479 45 45 45 62 62 62 F F F 24 480-499 47 47 47 64 64 64 T F T <--- 25 500-519 52 50 52 71 69 71 T T T <--- 26 520-539 48 48 48 67 67 67 F F F 27 540-559 50 50 50 69 69 69 F F F 28 560-579 50 50 50 69 69 69 F F F 29 580-599 51 51 51 70 70 70 F F F Max gen: 52, Max flush: 73 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 26 Verifying data now No change after verify Loop: 26 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 68 68 68 F F F 1 020-039 53 53 53 74 74 74 F F F 2 040-059 53 53 53 74 74 74 F F F 3 060-079 50 50 50 69 69 69 F F F 4 080-099 47 47 47 64 64 64 F F F 5 100-119 44 44 44 60 60 60 F F F 6 120-139 47 47 47 64 64 64 F F F 7 140-159 50 50 50 69 69 69 F F F 8 160-179 50 50 50 69 69 69 F F F 9 180-199 44 44 44 60 60 60 F F F 10 200-219 46 46 46 63 63 63 F F F 11 220-239 52 52 52 72 72 72 F F F 12 240-259 53 53 53 74 74 74 F F F 13 260-279 50 50 50 69 69 69 F F F 14 280-299 50 50 50 69 69 69 F F F 15 300-319 50 50 50 69 69 69 F F F 16 320-339 49 49 49 68 68 68 F F F 17 340-359 53 53 53 74 74 74 F F F 18 360-379 46 46 46 63 63 63 F F F 19 380-399 52 52 52 73 73 73 F F F 20 400-419 52 52 52 71 71 71 F F F 21 420-439 53 53 53 74 74 74 F F F 22 440-459 44 44 44 59 59 59 F F F 23 460-479 45 45 45 62 62 62 F F F 24 480-499 53 53 53 74 74 74 F F F 25 500-519 53 53 53 74 74 74 F F F 26 520-539 48 48 48 67 67 67 F F F 27 540-559 50 50 50 69 69 69 F F F 28 560-579 50 50 50 69 69 69 F F F 29 580-599 51 51 51 70 70 70 F F F Max gen: 53, Max flush: 74 Begin loop 27 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:28.348845412Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1352,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:21:28.355975253Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1352,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:28.356029903Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1352,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:28.362927484Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1352,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:28.363196374Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1352} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:28.363223654Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1352} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:28.363234294Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1352} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:28.363246114Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1352} {"msg":"Crucible 4ed8c5c8-46d9-4c09-805c-d337ebeb3891 has session id: c72c6ba3-5604-4f49-9a2c-68a1f044f678","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:28.363608545Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1352,"session_id":"c72c6ba3-5604-4f49-9a2c-68a1f044f678"} {"msg":"Upstairs opts: Upstairs UUID: 4ed8c5c8-46d9-4c09-805c-d337ebeb3891, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:28.363632155Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1352,"session_id":"c72c6ba3-5604-4f49-9a2c-68a1f044f678"} {"msg":"Crucible stats registered with UUID: 4ed8c5c8-46d9-4c09-805c-d337ebeb3891","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:28.363642145Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1352,"session_id":"c72c6ba3-5604-4f49-9a2c-68a1f044f678"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:21:28.363651325Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1352,"session_id":"c72c6ba3-5604-4f49-9a2c-68a1f044f678"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:28.363797875Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1352,"":"io task","client":"0","":"downstairs","session_id":"c72c6ba3-5604-4f49-9a2c-68a1f044f678"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:28.363815825Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1352,"":"io task","client":"2","":"downstairs","session_id":"c72c6ba3-5604-4f49-9a2c-68a1f044f678"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:28.363826195Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1352,"":"io task","client":"1","":"downstairs","session_id":"c72c6ba3-5604-4f49-9a2c-68a1f044f678"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:28.364032415Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1352,"task":"crutest"} {"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:28.364049775Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1352} {"msg":"4ed8c5c8-46d9-4c09-805c-d337ebeb3891 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:28.364082525Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1352,"session_id":"c72c6ba3-5604-4f49-9a2c-68a1f044f678"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:28.364091985Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1352,"client":"0","":"downstairs","session_id":"c72c6ba3-5604-4f49-9a2c-68a1f044f678"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:28.364101745Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1352,"client":"1","":"downstairs","session_id":"c72c6ba3-5604-4f49-9a2c-68a1f044f678"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:28.364111535Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1352,"client":"2","":"downstairs","session_id":"c72c6ba3-5604-4f49-9a2c-68a1f044f678"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:28.364121225Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1352,"":"io task","client":"2","":"downstairs","session_id":"c72c6ba3-5604-4f49-9a2c-68a1f044f678"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:28.364133295Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1352,"":"io task","client":"0","":"downstairs","session_id":"c72c6ba3-5604-4f49-9a2c-68a1f044f678"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:28.364143455Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1352,"":"io task","client":"1","":"downstairs","session_id":"c72c6ba3-5604-4f49-9a2c-68a1f044f678"} {"msg":"ds_connection connected from Ok(127.0.0.1:53930)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:28.364216295Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1352,"":"io task","client":"2","":"downstairs","session_id":"c72c6ba3-5604-4f49-9a2c-68a1f044f678"} {"msg":"ds_connection connected from Ok(127.0.0.1:58662)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:28.364227326Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1352,"":"io task","client":"1","":"downstairs","session_id":"c72c6ba3-5604-4f49-9a2c-68a1f044f678"} {"msg":"ds_connection connected from Ok(127.0.0.1:45945)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:28.364237746Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1352,"":"io task","client":"0","":"downstairs","session_id":"c72c6ba3-5604-4f49-9a2c-68a1f044f678"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:28.364905367Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1352,"client":"1","":"downstairs","session_id":"c72c6ba3-5604-4f49-9a2c-68a1f044f678"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:28.364918857Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1352,"client":"2","":"downstairs","session_id":"c72c6ba3-5604-4f49-9a2c-68a1f044f678"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:28.364929727Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1352,"client":"0","":"downstairs","session_id":"c72c6ba3-5604-4f49-9a2c-68a1f044f678"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:28.365249087Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1352,"session_id":"c72c6ba3-5604-4f49-9a2c-68a1f044f678"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:28.365259787Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1352,"session_id":"c72c6ba3-5604-4f49-9a2c-68a1f044f678"} {"msg":"[0]R flush_numbers[0..12]: [68, 74, 74, 69, 64, 60, 64, 69, 69, 60, 63, 72]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:28.365301887Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1352,"":"downstairs","":"downstairs","session_id":"c72c6ba3-5604-4f49-9a2c-68a1f044f678"} {"msg":"[0]R generation[0..12]: [49, 53, 53, 50, 47, 44, 47, 50, 50, 44, 46, 52]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:28.365313427Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1352,"":"downstairs","":"downstairs","session_id":"c72c6ba3-5604-4f49-9a2c-68a1f044f678"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:28.365324717Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1352,"":"downstairs","":"downstairs","session_id":"c72c6ba3-5604-4f49-9a2c-68a1f044f678"} {"msg":"[1]R flush_numbers[0..12]: [68, 74, 74, 69, 64, 60, 64, 69, 69, 60, 63, 72]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:28.365335367Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1352,"":"downstairs","":"downstairs","session_id":"c72c6ba3-5604-4f49-9a2c-68a1f044f678"} {"msg":"[1]R generation[0..12]: [49, 53, 53, 50, 47, 44, 47, 50, 50, 44, 46, 52]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:28.365347557Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1352,"":"downstairs","":"downstairs","session_id":"c72c6ba3-5604-4f49-9a2c-68a1f044f678"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:28.365358557Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1352,"":"downstairs","":"downstairs","session_id":"c72c6ba3-5604-4f49-9a2c-68a1f044f678"} {"msg":"[2]R flush_numbers[0..12]: [68, 74, 74, 69, 64, 60, 64, 69, 69, 60, 63, 72]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:28.365369017Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1352,"":"downstairs","":"downstairs","session_id":"c72c6ba3-5604-4f49-9a2c-68a1f044f678"} {"msg":"[2]R generation[0..12]: [49, 53, 53, 50, 47, 44, 47, 50, 50, 44, 46, 52]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:28.365379427Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1352,"":"downstairs","":"downstairs","session_id":"c72c6ba3-5604-4f49-9a2c-68a1f044f678"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:28.365390517Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1352,"":"downstairs","":"downstairs","session_id":"c72c6ba3-5604-4f49-9a2c-68a1f044f678"} {"msg":"Max found gen is 54","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:28.365400097Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1352,"":"downstairs","":"downstairs","session_id":"c72c6ba3-5604-4f49-9a2c-68a1f044f678"} {"msg":"Generation requested: 54 >= found:54","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:28.365409837Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1352,"":"downstairs","":"downstairs","session_id":"c72c6ba3-5604-4f49-9a2c-68a1f044f678"} {"msg":"Next flush: 75","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:28.365419397Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1352,"":"downstairs","":"downstairs","session_id":"c72c6ba3-5604-4f49-9a2c-68a1f044f678"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:28.365428887Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1352,"":"downstairs","":"downstairs","session_id":"c72c6ba3-5604-4f49-9a2c-68a1f044f678"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:28.365438937Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1352,"session_id":"c72c6ba3-5604-4f49-9a2c-68a1f044f678"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:28.365447957Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1352,"client":"0","":"downstairs","session_id":"c72c6ba3-5604-4f49-9a2c-68a1f044f678"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:28.365457987Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1352,"client":"1","":"downstairs","session_id":"c72c6ba3-5604-4f49-9a2c-68a1f044f678"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:28.365467987Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1352,"client":"2","":"downstairs","session_id":"c72c6ba3-5604-4f49-9a2c-68a1f044f678"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:28.365478267Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1352,"session_id":"c72c6ba3-5604-4f49-9a2c-68a1f044f678"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:28.365487637Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1352,"session_id":"c72c6ba3-5604-4f49-9a2c-68a1f044f678"} {"msg":"4ed8c5c8-46d9-4c09-805c-d337ebeb3891 is now active with session: c72c6ba3-5604-4f49-9a2c-68a1f044f678","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:28.365497577Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1352,"session_id":"c72c6ba3-5604-4f49-9a2c-68a1f044f678"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:28.365507537Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1352,"session_id":"c72c6ba3-5604-4f49-9a2c-68a1f044f678"} {"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:28.365516677Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1352} {"msg":"Activated sub_volume 4ed8c5c8-46d9-4c09-805c-d337ebeb3891","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:28.365524907Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1352} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:28.365533197Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1352,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:28.365857198Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1352,"task":"crutest"} 01/30 Write block 575 len 4096 data: 5 02/30 Write block 371 len 12288 data: 5 7 8 03/30 Write block 161 len 8192 data: 6 3 04/30 Read block 106 len 4096 05/30 Write block 376 len 4096 data: 9 06/30 Read block 459 len 12288 07/30 Write block 529 len 32768 data: 9 6 5 5 5 4 5 4 08/30 Read block 288 len 20480 09/30 Write block 126 len 4096 data: 4 10/30 Write block 347 len 16384 data: 7 7 6 4 11/30 Write block 331 len 12288 data: 6 5 4 12/30 Read block 405 len 8192 13/30 Write block 216 len 32768 data: 7 5 5 4 2 2 2 2 14/30 Write block 506 len 36864 data: 5 3 3 3 4 3 4 4 3 15/30 Read block 452 len 4096 16/30 Flush 17/30 Write block 525 len 16384 data: 8 7 9 10 18/30 Flush 19/30 Write block 326 len 12288 data: 8 8 9 20/30 Write block 184 len 4096 data: 5 21/30 Write block 338 len 16384 data: 4 5 6 7 22/30 Write block 263 len 24576 data: 7 6 7 8 9 10 23/30 Write block 467 len 20480 data: 5 8 7 6 6 24/30 Write block 165 len 36864 data: 3 3 3 3 4 4 4 4 4 25/30 Flush 26/30 Flush 27/30 Write block 103 len 8192 data: 4 5 28/30 Write block 295 len 32768 data: 7 7 6 7 8 6 9 9 29/30 Write block 204 len 36864 data: 7 7 5 6 6 7 8 9 8 30/30 Read block 99 len 16384 ---------------------------------------------------------------- Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Sent Done Done false 1001 Acked Write 3 Sent Done Done false 1002 Acked Write 2 Sent Done Done false 1003 Acked Read 1 Sent Done Done false 1004 Acked Write 1 Sent Done Done false 1005 Acked Read 3 Sent Done Done false 1006 Acked Write 8 Sent Done Done false 1007 Acked Read 5 Sent Done Done false 1008 Acked Write 1 Sent Done Done false 1009 Acked Write 4 Sent Done Done false 1010 Acked Write 3 Sent Done Done false 1011 Acked Read 2 Sent Done Done false 1012 Acked Write 8 Sent Done Done false 1013 Acked Write 9 Sent Done Done false 1014 Acked Read 1 Sent Done Done false 1015 Acked Flush 0 Sent Done Done false 1016 Acked Write 4 Sent Done Done false 1017 Acked Flush 0 Sent Done Done false 1018 Acked Write 3 Sent Done Done false 1019 Acked Write 1 Sent Done Done false 1020 Acked Write 4 Sent Done Done false 1021 Acked Write 6 Sent Done Done false 1022 Acked Write 5 Sent Done Done false 1023 Acked Write 9 Sent Done Done false 1024 Acked Flush 0 Sent Done Done false 1025 Acked Flush 0 Sent Done Done false 1026 Acked Write 2 Sent Done Done false 1027 Acked Write 8 Sent Done Done false 1028 Acked Write 9 Sent Done Done false 1029 Acked Read 4 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 0 1 31 Done 0 30 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:21:28.373440479Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1352} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:28.37361606Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1352,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 68 68 68 F F F 1 020-039 53 53 53 74 74 74 F F F 2 040-059 53 53 53 74 74 74 F F F 3 060-079 50 50 50 69 69 69 F F F 4 080-099 47 47 47 64 64 64 F F F 5 100-119 44 44 44 60 60 60 F T T <--- 6 120-139 47 54 54 64 75 75 F F F <--- 7 140-159 50 50 50 69 69 69 F F F 8 160-179 50 54 54 69 77 77 F F F <--- 9 180-199 44 54 54 60 77 77 F F F <--- 10 200-219 46 54 54 63 75 75 F T T <--- 11 220-239 52 54 54 72 75 75 F F F <--- 12 240-259 53 53 53 74 74 74 F F F 13 260-279 50 54 54 69 77 77 F F F <--- 14 280-299 50 50 50 69 69 69 F T T <--- 15 300-319 50 50 50 69 69 69 F T T <--- 16 320-339 49 54 54 68 77 77 F F F <--- 17 340-359 53 54 54 74 77 77 F F F <--- 18 360-379 46 54 54 63 75 75 F F F <--- 19 380-399 52 52 52 73 73 73 F F F 20 400-419 52 52 52 71 71 71 F F F 21 420-439 53 53 53 74 74 74 F F F 22 440-459 44 44 44 59 59 59 F F F 23 460-479 45 54 54 62 77 77 F F F <--- 24 480-499 53 53 53 74 74 74 F F F 25 500-519 53 54 54 74 75 75 F F F <--- 26 520-539 48 54 54 67 76 76 F F F <--- 27 540-559 50 50 50 69 69 69 F F F 28 560-579 50 54 54 69 75 75 F F F <--- 29 580-599 51 51 51 70 70 70 F F F Max gen: 54, Max flush: 77 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 27 Verifying data now No change after verify Loop: 27 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 68 68 68 F F F 1 020-039 53 53 53 74 74 74 F F F 2 040-059 53 53 53 74 74 74 F F F 3 060-079 50 50 50 69 69 69 F F F 4 080-099 47 47 47 64 64 64 F F F 5 100-119 55 55 55 78 78 78 F F F 6 120-139 54 54 54 75 75 75 F F F 7 140-159 50 50 50 69 69 69 F F F 8 160-179 54 54 54 77 77 77 F F F 9 180-199 54 54 54 77 77 77 F F F 10 200-219 55 55 55 78 78 78 F F F 11 220-239 54 54 54 75 75 75 F F F 12 240-259 53 53 53 74 74 74 F F F 13 260-279 54 54 54 77 77 77 F F F 14 280-299 55 55 55 78 78 78 F F F 15 300-319 55 55 55 78 78 78 F F F 16 320-339 54 54 54 77 77 77 F F F 17 340-359 54 54 54 77 77 77 F F F 18 360-379 54 54 54 75 75 75 F F F 19 380-399 52 52 52 73 73 73 F F F 20 400-419 52 52 52 71 71 71 F F F 21 420-439 53 53 53 74 74 74 F F F 22 440-459 44 44 44 59 59 59 F F F 23 460-479 54 54 54 77 77 77 F F F 24 480-499 53 53 53 74 74 74 F F F 25 500-519 54 54 54 75 75 75 F F F 26 520-539 54 54 54 76 76 76 F F F 27 540-559 50 50 50 69 69 69 F F F 28 560-579 54 54 54 75 75 75 F F F 29 580-599 51 51 51 70 70 70 F F F Max gen: 55, Max flush: 78 Begin loop 28 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:32.637971005Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1360,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:32.646243117Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1360,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:32.646543138Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1360} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:32.646568808Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1360} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:32.646577608Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1360} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:32.646589458Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1360} {"msg":"Crucible e99316b6-a4e4-4d66-ad2f-f21bfb5ad24e has session id: e64e614e-e3e2-4b06-b0aa-5a1e51c69b49","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:32.646991428Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1360,"session_id":"e64e614e-e3e2-4b06-b0aa-5a1e51c69b49"} {"msg":"Upstairs opts: Upstairs UUID: e99316b6-a4e4-4d66-ad2f-f21bfb5ad24e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:32.647019658Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1360,"session_id":"e64e614e-e3e2-4b06-b0aa-5a1e51c69b49"} {"msg":"Crucible stats registered with UUID: e99316b6-a4e4-4d66-ad2f-f21bfb5ad24e","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:32.647030488Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1360,"session_id":"e64e614e-e3e2-4b06-b0aa-5a1e51c69b49"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:21:32.647041798Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1360,"session_id":"e64e614e-e3e2-4b06-b0aa-5a1e51c69b49"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:32.647194829Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1360,"":"io task","client":"1","":"downstairs","session_id":"e64e614e-e3e2-4b06-b0aa-5a1e51c69b49"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:32.647225569Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1360,"":"io task","client":"0","":"downstairs","session_id":"e64e614e-e3e2-4b06-b0aa-5a1e51c69b49"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:32.647236429Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1360,"":"io task","client":"2","":"downstairs","session_id":"e64e614e-e3e2-4b06-b0aa-5a1e51c69b49"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:32.647342879Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1360,"task":"crutest"} {"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:32.647358499Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1360} {"msg":"e99316b6-a4e4-4d66-ad2f-f21bfb5ad24e active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:32.647397389Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1360,"session_id":"e64e614e-e3e2-4b06-b0aa-5a1e51c69b49"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:32.647406929Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1360,"client":"0","":"downstairs","session_id":"e64e614e-e3e2-4b06-b0aa-5a1e51c69b49"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:32.647416669Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1360,"client":"1","":"downstairs","session_id":"e64e614e-e3e2-4b06-b0aa-5a1e51c69b49"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:32.647426489Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1360,"client":"2","":"downstairs","session_id":"e64e614e-e3e2-4b06-b0aa-5a1e51c69b49"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:32.647436179Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1360,"":"io task","client":"2","":"downstairs","session_id":"e64e614e-e3e2-4b06-b0aa-5a1e51c69b49"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:32.647446159Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1360,"":"io task","client":"0","":"downstairs","session_id":"e64e614e-e3e2-4b06-b0aa-5a1e51c69b49"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:32.647456089Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1360,"":"io task","client":"1","":"downstairs","session_id":"e64e614e-e3e2-4b06-b0aa-5a1e51c69b49"} {"msg":"ds_connection connected from Ok(127.0.0.1:51423)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:32.647567219Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1360,"":"io task","client":"2","":"downstairs","session_id":"e64e614e-e3e2-4b06-b0aa-5a1e51c69b49"} {"msg":"ds_connection connected from Ok(127.0.0.1:56438)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:32.647596329Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1360,"":"io task","client":"0","":"downstairs","session_id":"e64e614e-e3e2-4b06-b0aa-5a1e51c69b49"} {"msg":"ds_connection connected from Ok(127.0.0.1:50019)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:32.647607639Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1360,"":"io task","client":"1","":"downstairs","session_id":"e64e614e-e3e2-4b06-b0aa-5a1e51c69b49"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:32.64825882Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1360,"client":"2","":"downstairs","session_id":"e64e614e-e3e2-4b06-b0aa-5a1e51c69b49"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:32.64828915Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1360,"client":"0","":"downstairs","session_id":"e64e614e-e3e2-4b06-b0aa-5a1e51c69b49"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:32.64830059Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1360,"client":"1","":"downstairs","session_id":"e64e614e-e3e2-4b06-b0aa-5a1e51c69b49"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:32.648597311Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1360,"session_id":"e64e614e-e3e2-4b06-b0aa-5a1e51c69b49"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:32.648607191Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1360,"session_id":"e64e614e-e3e2-4b06-b0aa-5a1e51c69b49"} {"msg":"[0]R flush_numbers[0..12]: [68, 74, 74, 69, 64, 78, 75, 69, 77, 77, 78, 75]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:32.648641051Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1360,"":"downstairs","":"downstairs","session_id":"e64e614e-e3e2-4b06-b0aa-5a1e51c69b49"} {"msg":"[0]R generation[0..12]: [49, 53, 53, 50, 47, 55, 54, 50, 54, 54, 55, 54]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:32.648652151Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1360,"":"downstairs","":"downstairs","session_id":"e64e614e-e3e2-4b06-b0aa-5a1e51c69b49"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:32.648663341Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1360,"":"downstairs","":"downstairs","session_id":"e64e614e-e3e2-4b06-b0aa-5a1e51c69b49"} {"msg":"[1]R flush_numbers[0..12]: [68, 74, 74, 69, 64, 78, 75, 69, 77, 77, 78, 75]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:32.648673841Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1360,"":"downstairs","":"downstairs","session_id":"e64e614e-e3e2-4b06-b0aa-5a1e51c69b49"} {"msg":"[1]R generation[0..12]: [49, 53, 53, 50, 47, 55, 54, 50, 54, 54, 55, 54]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:32.648684321Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1360,"":"downstairs","":"downstairs","session_id":"e64e614e-e3e2-4b06-b0aa-5a1e51c69b49"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:32.648695221Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1360,"":"downstairs","":"downstairs","session_id":"e64e614e-e3e2-4b06-b0aa-5a1e51c69b49"} {"msg":"[2]R flush_numbers[0..12]: [68, 74, 74, 69, 64, 78, 75, 69, 77, 77, 78, 75]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:32.648705701Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1360,"":"downstairs","":"downstairs","session_id":"e64e614e-e3e2-4b06-b0aa-5a1e51c69b49"} {"msg":"[2]R generation[0..12]: [49, 53, 53, 50, 47, 55, 54, 50, 54, 54, 55, 54]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:32.648716151Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1360,"":"downstairs","":"downstairs","session_id":"e64e614e-e3e2-4b06-b0aa-5a1e51c69b49"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:32.648727161Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1360,"":"downstairs","":"downstairs","session_id":"e64e614e-e3e2-4b06-b0aa-5a1e51c69b49"} {"msg":"Max found gen is 56","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:32.648738961Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1360,"":"downstairs","":"downstairs","session_id":"e64e614e-e3e2-4b06-b0aa-5a1e51c69b49"} {"msg":"Generation requested: 56 >= found:56","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:32.648748831Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1360,"":"downstairs","":"downstairs","session_id":"e64e614e-e3e2-4b06-b0aa-5a1e51c69b49"} {"msg":"Next flush: 79","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:32.648758261Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1360,"":"downstairs","":"downstairs","session_id":"e64e614e-e3e2-4b06-b0aa-5a1e51c69b49"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:32.648767771Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1360,"":"downstairs","":"downstairs","session_id":"e64e614e-e3e2-4b06-b0aa-5a1e51c69b49"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:32.648777691Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1360,"session_id":"e64e614e-e3e2-4b06-b0aa-5a1e51c69b49"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:32.648786511Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1360,"client":"0","":"downstairs","session_id":"e64e614e-e3e2-4b06-b0aa-5a1e51c69b49"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:32.648796161Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1360,"client":"1","":"downstairs","session_id":"e64e614e-e3e2-4b06-b0aa-5a1e51c69b49"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:32.648805831Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1360,"client":"2","":"downstairs","session_id":"e64e614e-e3e2-4b06-b0aa-5a1e51c69b49"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:32.648815721Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1360,"session_id":"e64e614e-e3e2-4b06-b0aa-5a1e51c69b49"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:32.648824991Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1360,"session_id":"e64e614e-e3e2-4b06-b0aa-5a1e51c69b49"} {"msg":"e99316b6-a4e4-4d66-ad2f-f21bfb5ad24e is now active with session: e64e614e-e3e2-4b06-b0aa-5a1e51c69b49","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:32.648835081Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1360,"session_id":"e64e614e-e3e2-4b06-b0aa-5a1e51c69b49"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:32.648844061Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1360,"session_id":"e64e614e-e3e2-4b06-b0aa-5a1e51c69b49"} {"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:32.648853281Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1360} {"msg":"Activated sub_volume e99316b6-a4e4-4d66-ad2f-f21bfb5ad24e","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:32.648861351Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1360} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:32.648869531Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1360,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:32.649175602Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1360,"task":"crutest"} 01/30 Write block 592 len 8192 data: 2 3 02/30 Write block 200 len 4096 data: 5 03/30 Read block 24 len 28672 04/30 Read block 508 len 20480 05/30 Read block 65 len 12288 06/30 Flush 07/30 Write block 225 len 8192 data: 2 2 08/30 Write block 506 len 28672 data: 6 4 4 4 5 4 5 09/30 Write block 30 len 8192 data: 6 6 10/30 Read block 281 len 32768 11/30 Read block 281 len 8192 12/30 Flush 13/30 Write block 225 len 16384 data: 3 3 2 5 14/30 Read block 391 len 8192 15/30 Read block 390 len 20480 16/30 Write block 102 len 28672 data: 3 5 6 6 6 5 5 17/30 Read block 189 len 16384 18/30 Flush 19/30 Write block 489 len 8192 data: 7 6 20/30 Write block 407 len 28672 data: 4 5 4 4 2 3 3 21/30 Read block 288 len 36864 22/30 Write block 102 len 40960 data: 4 6 7 7 7 6 6 6 7 6 23/30 Flush 24/30 Write block 362 len 4096 data: 8 25/30 Write block 583 len 28672 data: 3 3 3 3 3 3 3 26/30 Read block 550 len 28672 27/30 Read block 456 len 16384 28/30 Read block 484 len 8192 29/30 Write block 55 len 4096 data: 5 30/30 Write block 270 len 12288 data: 12 11 11 ---------------------------------------------------------------- Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Write 1 Done Sent Done false 1002 Acked Read 7 Done Sent Done false 1003 Acked Read 5 Done Sent Done false 1004 Acked Read 3 Done Sent Done false 1005 Acked Flush 0 Done Sent Done false 1006 Acked Write 2 Done Sent Done false 1007 Acked Write 7 Done Sent Done false 1008 Acked Write 2 Done Sent Done false 1009 Acked Read 8 Done Sent Done false 1010 Acked Read 2 Done Sent Done false 1011 Acked Flush 0 Done Sent Done false 1012 Acked Write 4 Done Sent Done false 1013 Acked Read 2 Done Sent Done false 1014 Acked Read 5 Done Sent Done false 1015 Acked Write 7 Done Sent Done false 1016 Acked Read 4 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Write 2 Done Sent Done false 1019 Acked Write 7 Done Sent Done false 1020 Acked Read 9 Done Sent Done false 1021 Acked Write 10 Done Sent Done false 1022 Acked Flush 0 Done Sent Done false 1023 Acked Write 1 Done Sent Done false 1024 Acked Write 7 Done Sent Done false 1025 Acked Read 7 Done Sent Done false 1026 Acked Read 4 Done Sent Done false 1027 Acked Read 2 Done Sent Done false 1028 Acked Write 1 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 29 2 33 Done 28 1 28 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) None Some(JobId(1022)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:21:32.657443304Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1360} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:32.657628224Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1360,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 68 68 68 F F F 1 020-039 56 53 56 80 74 80 F F F <--- 2 040-059 53 53 53 74 74 74 T F T <--- 3 060-079 50 50 50 69 69 69 F F F 4 080-099 47 47 47 64 64 64 F F F 5 100-119 56 55 56 82 78 82 F F F <--- 6 120-139 54 54 54 75 75 75 F F F 7 140-159 50 50 50 69 69 69 F F F 8 160-179 54 54 54 77 77 77 F F F 9 180-199 54 54 54 77 77 77 F F F 10 200-219 56 55 56 79 78 79 F F F <--- 11 220-239 56 54 56 81 75 81 F F F <--- 12 240-259 53 53 53 74 74 74 F F F 13 260-279 54 54 54 77 77 77 T F T <--- 14 280-299 55 55 55 78 78 78 F F F 15 300-319 55 55 55 78 78 78 F F F 16 320-339 54 54 54 77 77 77 F F F 17 340-359 54 54 54 77 77 77 F F F 18 360-379 54 54 54 75 75 75 T F T <--- 19 380-399 52 52 52 73 73 73 F F F 20 400-419 56 52 56 82 71 82 F F F <--- 21 420-439 53 53 53 74 74 74 F F F 22 440-459 44 44 44 59 59 59 F F F 23 460-479 54 54 54 77 77 77 F F F 24 480-499 56 53 56 82 74 82 F F F <--- 25 500-519 56 54 56 80 75 80 F F F <--- 26 520-539 54 54 54 76 76 76 F F F 27 540-559 50 50 50 69 69 69 F F F 28 560-579 54 54 54 75 75 75 F F F 29 580-599 56 51 56 79 70 79 T T T <--- Max gen: 56, Max flush: 82 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 28 Verifying data now No change after verify Loop: 28 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 68 68 68 F F F 1 020-039 56 56 56 80 80 80 F F F 2 040-059 57 57 57 83 83 83 F F F 3 060-079 50 50 50 69 69 69 F F F 4 080-099 47 47 47 64 64 64 F F F 5 100-119 56 56 56 82 82 82 F F F 6 120-139 54 54 54 75 75 75 F F F 7 140-159 50 50 50 69 69 69 F F F 8 160-179 54 54 54 77 77 77 F F F 9 180-199 54 54 54 77 77 77 F F F 10 200-219 56 56 56 79 79 79 F F F 11 220-239 56 56 56 81 81 81 F F F 12 240-259 53 53 53 74 74 74 F F F 13 260-279 57 57 57 83 83 83 F F F 14 280-299 55 55 55 78 78 78 F F F 15 300-319 55 55 55 78 78 78 F F F 16 320-339 54 54 54 77 77 77 F F F 17 340-359 54 54 54 77 77 77 F F F 18 360-379 57 57 57 83 83 83 F F F 19 380-399 52 52 52 73 73 73 F F F 20 400-419 56 56 56 82 82 82 F F F 21 420-439 53 53 53 74 74 74 F F F 22 440-459 44 44 44 59 59 59 F F F 23 460-479 54 54 54 77 77 77 F F F 24 480-499 56 56 56 82 82 82 F F F 25 500-519 56 56 56 80 80 80 F F F 26 520-539 54 54 54 76 76 76 F F F 27 540-559 50 50 50 69 69 69 F F F 28 560-579 54 54 54 75 75 75 F F F 29 580-599 57 57 57 83 83 83 F F F Max gen: 57, Max flush: 83 Begin loop 29 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:36.904052662Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1368,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:36.912167574Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1368,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:36.912465405Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1368} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:36.912493835Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1368} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:36.912502285Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1368} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:36.912514085Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1368} {"msg":"Crucible 9ef8e39a-601e-4b80-b54f-92d841b482fb has session id: 99118ce3-07af-4ea4-aa49-ee67021bb2ac","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:36.912889345Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1368,"session_id":"99118ce3-07af-4ea4-aa49-ee67021bb2ac"} {"msg":"Upstairs opts: Upstairs UUID: 9ef8e39a-601e-4b80-b54f-92d841b482fb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:36.912918975Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1368,"session_id":"99118ce3-07af-4ea4-aa49-ee67021bb2ac"} {"msg":"Crucible stats registered with UUID: 9ef8e39a-601e-4b80-b54f-92d841b482fb","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:36.912929795Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1368,"session_id":"99118ce3-07af-4ea4-aa49-ee67021bb2ac"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:21:36.912939185Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1368,"session_id":"99118ce3-07af-4ea4-aa49-ee67021bb2ac"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:36.913077036Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1368,"":"io task","client":"1","":"downstairs","session_id":"99118ce3-07af-4ea4-aa49-ee67021bb2ac"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:36.913108636Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1368,"":"io task","client":"0","":"downstairs","session_id":"99118ce3-07af-4ea4-aa49-ee67021bb2ac"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:36.913119536Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1368,"":"io task","client":"2","":"downstairs","session_id":"99118ce3-07af-4ea4-aa49-ee67021bb2ac"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:36.913289296Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1368,"task":"crutest"} {"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:36.913302166Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1368} {"msg":"9ef8e39a-601e-4b80-b54f-92d841b482fb active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:36.913318046Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1368,"session_id":"99118ce3-07af-4ea4-aa49-ee67021bb2ac"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:36.913327036Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1368,"client":"0","":"downstairs","session_id":"99118ce3-07af-4ea4-aa49-ee67021bb2ac"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:36.913338776Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1368,"client":"1","":"downstairs","session_id":"99118ce3-07af-4ea4-aa49-ee67021bb2ac"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:36.913348766Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1368,"client":"2","":"downstairs","session_id":"99118ce3-07af-4ea4-aa49-ee67021bb2ac"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:36.913358406Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1368,"":"io task","client":"2","":"downstairs","session_id":"99118ce3-07af-4ea4-aa49-ee67021bb2ac"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:36.913368356Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1368,"":"io task","client":"0","":"downstairs","session_id":"99118ce3-07af-4ea4-aa49-ee67021bb2ac"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:36.913378446Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1368,"":"io task","client":"1","":"downstairs","session_id":"99118ce3-07af-4ea4-aa49-ee67021bb2ac"} {"msg":"ds_connection connected from Ok(127.0.0.1:44123)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:36.913442266Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1368,"":"io task","client":"2","":"downstairs","session_id":"99118ce3-07af-4ea4-aa49-ee67021bb2ac"} {"msg":"ds_connection connected from Ok(127.0.0.1:59739)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:36.913453246Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1368,"":"io task","client":"0","":"downstairs","session_id":"99118ce3-07af-4ea4-aa49-ee67021bb2ac"} {"msg":"ds_connection connected from Ok(127.0.0.1:41696)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:36.913471186Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1368,"":"io task","client":"1","":"downstairs","session_id":"99118ce3-07af-4ea4-aa49-ee67021bb2ac"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:36.914155977Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1368,"client":"2","":"downstairs","session_id":"99118ce3-07af-4ea4-aa49-ee67021bb2ac"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:36.914196667Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1368,"client":"1","":"downstairs","session_id":"99118ce3-07af-4ea4-aa49-ee67021bb2ac"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:36.914223957Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1368,"client":"0","":"downstairs","session_id":"99118ce3-07af-4ea4-aa49-ee67021bb2ac"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:36.914488768Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1368,"session_id":"99118ce3-07af-4ea4-aa49-ee67021bb2ac"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:36.914512918Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1368,"session_id":"99118ce3-07af-4ea4-aa49-ee67021bb2ac"} {"msg":"[0]R flush_numbers[0..12]: [68, 80, 83, 69, 64, 82, 75, 69, 77, 77, 79, 81]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:36.914559778Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1368,"":"downstairs","":"downstairs","session_id":"99118ce3-07af-4ea4-aa49-ee67021bb2ac"} {"msg":"[0]R generation[0..12]: [49, 56, 57, 50, 47, 56, 54, 50, 54, 54, 56, 56]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:36.914571538Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1368,"":"downstairs","":"downstairs","session_id":"99118ce3-07af-4ea4-aa49-ee67021bb2ac"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:36.914584768Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1368,"":"downstairs","":"downstairs","session_id":"99118ce3-07af-4ea4-aa49-ee67021bb2ac"} {"msg":"[1]R flush_numbers[0..12]: [68, 80, 83, 69, 64, 82, 75, 69, 77, 77, 79, 81]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:36.914595478Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1368,"":"downstairs","":"downstairs","session_id":"99118ce3-07af-4ea4-aa49-ee67021bb2ac"} {"msg":"[1]R generation[0..12]: [49, 56, 57, 50, 47, 56, 54, 50, 54, 54, 56, 56]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:36.914605828Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1368,"":"downstairs","":"downstairs","session_id":"99118ce3-07af-4ea4-aa49-ee67021bb2ac"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:36.914616888Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1368,"":"downstairs","":"downstairs","session_id":"99118ce3-07af-4ea4-aa49-ee67021bb2ac"} {"msg":"[2]R flush_numbers[0..12]: [68, 80, 83, 69, 64, 82, 75, 69, 77, 77, 79, 81]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:36.914627278Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1368,"":"downstairs","":"downstairs","session_id":"99118ce3-07af-4ea4-aa49-ee67021bb2ac"} {"msg":"[2]R generation[0..12]: [49, 56, 57, 50, 47, 56, 54, 50, 54, 54, 56, 56]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:36.914637888Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1368,"":"downstairs","":"downstairs","session_id":"99118ce3-07af-4ea4-aa49-ee67021bb2ac"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:36.914648978Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1368,"":"downstairs","":"downstairs","session_id":"99118ce3-07af-4ea4-aa49-ee67021bb2ac"} {"msg":"Max found gen is 58","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:36.914658438Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1368,"":"downstairs","":"downstairs","session_id":"99118ce3-07af-4ea4-aa49-ee67021bb2ac"} {"msg":"Generation requested: 58 >= found:58","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:36.914668178Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1368,"":"downstairs","":"downstairs","session_id":"99118ce3-07af-4ea4-aa49-ee67021bb2ac"} {"msg":"Next flush: 84","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:36.914677548Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1368,"":"downstairs","":"downstairs","session_id":"99118ce3-07af-4ea4-aa49-ee67021bb2ac"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:36.914687138Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1368,"":"downstairs","":"downstairs","session_id":"99118ce3-07af-4ea4-aa49-ee67021bb2ac"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:36.914696888Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1368,"session_id":"99118ce3-07af-4ea4-aa49-ee67021bb2ac"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:36.914705688Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1368,"client":"0","":"downstairs","session_id":"99118ce3-07af-4ea4-aa49-ee67021bb2ac"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:36.914715478Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1368,"client":"1","":"downstairs","session_id":"99118ce3-07af-4ea4-aa49-ee67021bb2ac"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:36.914726448Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1368,"client":"2","":"downstairs","session_id":"99118ce3-07af-4ea4-aa49-ee67021bb2ac"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:36.914736358Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1368,"session_id":"99118ce3-07af-4ea4-aa49-ee67021bb2ac"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:36.914745598Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1368,"session_id":"99118ce3-07af-4ea4-aa49-ee67021bb2ac"} {"msg":"9ef8e39a-601e-4b80-b54f-92d841b482fb is now active with session: 99118ce3-07af-4ea4-aa49-ee67021bb2ac","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:36.914755698Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1368,"session_id":"99118ce3-07af-4ea4-aa49-ee67021bb2ac"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:36.914764608Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1368,"session_id":"99118ce3-07af-4ea4-aa49-ee67021bb2ac"} {"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:36.914773798Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1368} {"msg":"Activated sub_volume 9ef8e39a-601e-4b80-b54f-92d841b482fb","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:36.914782128Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1368} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:36.914790358Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1368,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:36.915112199Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1368,"task":"crutest"} 01/30 Write block 121 len 20480 data: 6 6 6 5 5 02/30 Write block 44 len 4096 data: 2 03/30 Read block 390 len 4096 04/30 Write block 244 len 20480 data: 6 8 7 7 8 05/30 Read block 334 len 12288 06/30 Read block 350 len 16384 07/30 Write block 189 len 20480 data: 5 5 6 6 6 08/30 Read block 103 len 24576 09/30 Read block 294 len 20480 10/30 Read block 392 len 20480 11/30 Write block 587 len 24576 data: 4 4 4 3 2 3 12/30 Read block 177 len 8192 13/30 Write block 566 len 28672 data: 4 5 4 6 6 6 6 14/30 Read block 331 len 20480 15/30 Read block 521 len 40960 16/30 Write block 457 len 12288 data: 5 7 6 17/30 Read block 370 len 28672 18/30 Write block 421 len 8192 data: 5 6 19/30 Write block 364 len 36864 data: 7 7 7 5 5 4 5 6 8 20/30 Flush 21/30 Write block 582 len 24576 data: 3 4 4 4 4 5 22/30 Read block 246 len 36864 23/30 Read block 82 len 40960 24/30 Read block 376 len 12288 25/30 Read block 362 len 20480 26/30 Read block 30 len 8192 27/30 Write block 586 len 12288 data: 5 6 5 28/30 Write block 492 len 24576 data: 7 7 5 4 4 4 29/30 Read block 354 len 36864 30/30 Read block 117 len 40960 ---------------------------------------------------------------- Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Sent Done false 1001 Acked Write 1 Done Sent Done false 1002 Acked Read 1 Done Sent Done false 1003 Acked Write 5 Done Sent Done false 1004 Acked Read 3 Done Sent Done false 1005 Acked Read 4 Done Sent Done false 1006 Acked Write 5 Done Sent Done false 1007 Acked Read 6 Done Sent Done false 1008 Acked Read 5 Done Sent Done false 1009 Acked Read 5 Done Sent Done false 1010 Acked Write 6 Done Sent Done false 1011 Acked Read 2 Done Sent Done false 1012 Acked Write 7 Done Sent Done false 1013 Acked Read 5 Done Sent Done false 1014 Acked Read 10 Done Sent Done false 1015 Acked Write 3 Done Sent Done false 1016 Acked Read 7 Done Sent Done false 1017 Acked Write 2 Done Sent Done false 1018 Acked Write 9 Done Sent Done false 1019 Acked Flush 0 Done Sent Done false 1020 Acked Write 6 Done Sent Done false 1021 Acked Read 9 Done Sent Done false 1022 Acked Read 10 Done Sent Done false 1023 Acked Read 3 Done Sent Done false 1024 Acked Read 5 Done Sent Done false 1025 Acked Read 2 Done Sent Done false 1026 Acked Write 3 Done Sent Done false 1027 Acked Write 6 Done Sent Done false 1028 Acked Read 9 Done Sent Done false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 30 1 31 Done 30 0 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) None Some(JobId(1019)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:21:36.924458023Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1368} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:36.924635273Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1368,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 68 68 68 F F F 1 020-039 56 56 56 80 80 80 F F F 2 040-059 58 57 58 84 83 84 F F F <--- 3 060-079 50 50 50 69 69 69 F F F 4 080-099 47 47 47 64 64 64 F F F 5 100-119 56 56 56 82 82 82 F F F 6 120-139 58 54 58 84 75 84 F F F <--- 7 140-159 50 50 50 69 69 69 F F F 8 160-179 54 54 54 77 77 77 F F F 9 180-199 58 54 58 84 77 84 F F F <--- 10 200-219 56 56 56 79 79 79 F F F 11 220-239 56 56 56 81 81 81 F F F 12 240-259 58 53 58 84 74 84 F F F <--- 13 260-279 57 57 57 83 83 83 F F F 14 280-299 55 55 55 78 78 78 F F F 15 300-319 55 55 55 78 78 78 F F F 16 320-339 54 54 54 77 77 77 F F F 17 340-359 54 54 54 77 77 77 F F F 18 360-379 58 57 58 84 83 84 F F F <--- 19 380-399 52 52 52 73 73 73 F F F 20 400-419 56 56 56 82 82 82 F F F 21 420-439 58 53 58 84 74 84 F F F <--- 22 440-459 58 44 58 84 59 84 F F F <--- 23 460-479 54 54 54 77 77 77 F F F 24 480-499 56 56 56 82 82 82 T F T <--- 25 500-519 56 56 56 80 80 80 F F F 26 520-539 54 54 54 76 76 76 F F F 27 540-559 50 50 50 69 69 69 F F F 28 560-579 58 54 58 84 75 84 F F F <--- 29 580-599 58 57 58 84 83 84 T F T <--- Max gen: 58, Max flush: 84 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 29 Verifying data now No change after verify Loop: 29 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 68 68 68 F F F 1 020-039 56 56 56 80 80 80 F F F 2 040-059 58 58 58 84 84 84 F F F 3 060-079 50 50 50 69 69 69 F F F 4 080-099 47 47 47 64 64 64 F F F 5 100-119 56 56 56 82 82 82 F F F 6 120-139 58 58 58 84 84 84 F F F 7 140-159 50 50 50 69 69 69 F F F 8 160-179 54 54 54 77 77 77 F F F 9 180-199 58 58 58 84 84 84 F F F 10 200-219 56 56 56 79 79 79 F F F 11 220-239 56 56 56 81 81 81 F F F 12 240-259 58 58 58 84 84 84 F F F 13 260-279 57 57 57 83 83 83 F F F 14 280-299 55 55 55 78 78 78 F F F 15 300-319 55 55 55 78 78 78 F F F 16 320-339 54 54 54 77 77 77 F F F 17 340-359 54 54 54 77 77 77 F F F 18 360-379 58 58 58 84 84 84 F F F 19 380-399 52 52 52 73 73 73 F F F 20 400-419 56 56 56 82 82 82 F F F 21 420-439 58 58 58 84 84 84 F F F 22 440-459 58 58 58 84 84 84 F F F 23 460-479 54 54 54 77 77 77 F F F 24 480-499 59 59 59 85 85 85 F F F 25 500-519 56 56 56 80 80 80 F F F 26 520-539 54 54 54 76 76 76 F F F 27 540-559 50 50 50 69 69 69 F F F 28 560-579 58 58 58 84 84 84 F F F 29 580-599 59 59 59 85 85 85 F F F Max gen: 59, Max flush: 85 Begin loop 30 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:41.165988853Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1376,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:41.174177106Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1376,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:41.174487646Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1376} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:41.174514676Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1376} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:41.174523176Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1376} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:41.174534906Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1376} {"msg":"Crucible f01ef7e0-33b9-4c52-b96c-7e6aaa0b4bf3 has session id: b67916ac-b54f-4c03-8de1-bf874ab1f72a","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:41.174861307Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1376,"session_id":"b67916ac-b54f-4c03-8de1-bf874ab1f72a"} {"msg":"Upstairs opts: Upstairs UUID: f01ef7e0-33b9-4c52-b96c-7e6aaa0b4bf3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:41.174912827Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1376,"session_id":"b67916ac-b54f-4c03-8de1-bf874ab1f72a"} {"msg":"Crucible stats registered with UUID: f01ef7e0-33b9-4c52-b96c-7e6aaa0b4bf3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:41.174940157Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1376,"session_id":"b67916ac-b54f-4c03-8de1-bf874ab1f72a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:21:41.174950297Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1376,"session_id":"b67916ac-b54f-4c03-8de1-bf874ab1f72a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:41.175085857Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1376,"":"io task","client":"0","":"downstairs","session_id":"b67916ac-b54f-4c03-8de1-bf874ab1f72a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:41.175116307Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1376,"":"io task","client":"1","":"downstairs","session_id":"b67916ac-b54f-4c03-8de1-bf874ab1f72a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:41.175127197Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1376,"":"io task","client":"2","":"downstairs","session_id":"b67916ac-b54f-4c03-8de1-bf874ab1f72a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:41.175252577Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1376,"task":"crutest"} {"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:41.175266667Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1376} {"msg":"f01ef7e0-33b9-4c52-b96c-7e6aaa0b4bf3 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:41.175298487Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1376,"session_id":"b67916ac-b54f-4c03-8de1-bf874ab1f72a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:41.175307957Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1376,"client":"0","":"downstairs","session_id":"b67916ac-b54f-4c03-8de1-bf874ab1f72a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:41.175317627Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1376,"client":"1","":"downstairs","session_id":"b67916ac-b54f-4c03-8de1-bf874ab1f72a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:41.175327167Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1376,"client":"2","":"downstairs","session_id":"b67916ac-b54f-4c03-8de1-bf874ab1f72a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:41.175336577Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1376,"":"io task","client":"2","":"downstairs","session_id":"b67916ac-b54f-4c03-8de1-bf874ab1f72a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:41.175346567Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1376,"":"io task","client":"0","":"downstairs","session_id":"b67916ac-b54f-4c03-8de1-bf874ab1f72a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:41.175356587Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1376,"":"io task","client":"1","":"downstairs","session_id":"b67916ac-b54f-4c03-8de1-bf874ab1f72a"} {"msg":"ds_connection connected from Ok(127.0.0.1:65305)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:41.175538258Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1376,"":"io task","client":"1","":"downstairs","session_id":"b67916ac-b54f-4c03-8de1-bf874ab1f72a"} {"msg":"ds_connection connected from Ok(127.0.0.1:48219)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:41.175556448Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1376,"":"io task","client":"2","":"downstairs","session_id":"b67916ac-b54f-4c03-8de1-bf874ab1f72a"} {"msg":"ds_connection connected from Ok(127.0.0.1:62690)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:41.175566988Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1376,"":"io task","client":"0","":"downstairs","session_id":"b67916ac-b54f-4c03-8de1-bf874ab1f72a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:41.176114529Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1376,"client":"0","":"downstairs","session_id":"b67916ac-b54f-4c03-8de1-bf874ab1f72a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:41.176189119Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1376,"client":"1","":"downstairs","session_id":"b67916ac-b54f-4c03-8de1-bf874ab1f72a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:41.176227109Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1376,"client":"2","":"downstairs","session_id":"b67916ac-b54f-4c03-8de1-bf874ab1f72a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:41.176488179Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1376,"session_id":"b67916ac-b54f-4c03-8de1-bf874ab1f72a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:41.176510589Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1376,"session_id":"b67916ac-b54f-4c03-8de1-bf874ab1f72a"} {"msg":"[0]R flush_numbers[0..12]: [68, 80, 84, 69, 64, 82, 84, 69, 77, 84, 79, 81]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:41.176520889Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1376,"":"downstairs","":"downstairs","session_id":"b67916ac-b54f-4c03-8de1-bf874ab1f72a"} {"msg":"[0]R generation[0..12]: [49, 56, 58, 50, 47, 56, 58, 50, 54, 58, 56, 56]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:41.176531919Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1376,"":"downstairs","":"downstairs","session_id":"b67916ac-b54f-4c03-8de1-bf874ab1f72a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:41.176543299Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1376,"":"downstairs","":"downstairs","session_id":"b67916ac-b54f-4c03-8de1-bf874ab1f72a"} {"msg":"[1]R flush_numbers[0..12]: [68, 80, 84, 69, 64, 82, 84, 69, 77, 84, 79, 81]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:41.176553979Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1376,"":"downstairs","":"downstairs","session_id":"b67916ac-b54f-4c03-8de1-bf874ab1f72a"} {"msg":"[1]R generation[0..12]: [49, 56, 58, 50, 47, 56, 58, 50, 54, 58, 56, 56]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:41.176564559Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1376,"":"downstairs","":"downstairs","session_id":"b67916ac-b54f-4c03-8de1-bf874ab1f72a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:41.176575699Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1376,"":"downstairs","":"downstairs","session_id":"b67916ac-b54f-4c03-8de1-bf874ab1f72a"} {"msg":"[2]R flush_numbers[0..12]: [68, 80, 84, 69, 64, 82, 84, 69, 77, 84, 79, 81]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:41.176586259Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1376,"":"downstairs","":"downstairs","session_id":"b67916ac-b54f-4c03-8de1-bf874ab1f72a"} {"msg":"[2]R generation[0..12]: [49, 56, 58, 50, 47, 56, 58, 50, 54, 58, 56, 56]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:41.176598719Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1376,"":"downstairs","":"downstairs","session_id":"b67916ac-b54f-4c03-8de1-bf874ab1f72a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:41.176609819Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1376,"":"downstairs","":"downstairs","session_id":"b67916ac-b54f-4c03-8de1-bf874ab1f72a"} {"msg":"Max found gen is 60","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:41.176619169Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1376,"":"downstairs","":"downstairs","session_id":"b67916ac-b54f-4c03-8de1-bf874ab1f72a"} {"msg":"Generation requested: 60 >= found:60","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:41.176628849Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1376,"":"downstairs","":"downstairs","session_id":"b67916ac-b54f-4c03-8de1-bf874ab1f72a"} {"msg":"Next flush: 86","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:41.176638219Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1376,"":"downstairs","":"downstairs","session_id":"b67916ac-b54f-4c03-8de1-bf874ab1f72a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:41.176647539Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1376,"":"downstairs","":"downstairs","session_id":"b67916ac-b54f-4c03-8de1-bf874ab1f72a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:41.176657629Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1376,"session_id":"b67916ac-b54f-4c03-8de1-bf874ab1f72a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:41.176666659Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1376,"client":"0","":"downstairs","session_id":"b67916ac-b54f-4c03-8de1-bf874ab1f72a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:41.176676789Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1376,"client":"1","":"downstairs","session_id":"b67916ac-b54f-4c03-8de1-bf874ab1f72a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:41.176686639Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1376,"client":"2","":"downstairs","session_id":"b67916ac-b54f-4c03-8de1-bf874ab1f72a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:41.176696929Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1376,"session_id":"b67916ac-b54f-4c03-8de1-bf874ab1f72a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:41.176706219Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1376,"session_id":"b67916ac-b54f-4c03-8de1-bf874ab1f72a"} {"msg":"f01ef7e0-33b9-4c52-b96c-7e6aaa0b4bf3 is now active with session: b67916ac-b54f-4c03-8de1-bf874ab1f72a","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:41.176716339Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1376,"session_id":"b67916ac-b54f-4c03-8de1-bf874ab1f72a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:41.176725189Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1376,"session_id":"b67916ac-b54f-4c03-8de1-bf874ab1f72a"} {"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:41.176734339Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1376} {"msg":"Activated sub_volume f01ef7e0-33b9-4c52-b96c-7e6aaa0b4bf3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:41.17674226Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1376} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:41.17675068Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1376,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:41.17705507Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1376,"task":"crutest"} 01/30 Flush 02/30 Write block 259 len 28672 data: 5 5 4 5 8 7 8 03/30 Read block 143 len 32768 04/30 Write block 348 len 8192 data: 8 7 05/30 Read block 433 len 28672 06/30 Write block 483 len 28672 data: 5 6 6 6 6 5 8 07/30 Read block 93 len 36864 08/30 Read block 109 len 4096 09/30 Read block 302 len 32768 10/30 Read block 182 len 40960 11/30 Read block 71 len 24576 12/30 Read block 315 len 16384 13/30 Write block 369 len 4096 data: 5 14/30 Read block 39 len 12288 15/30 Write block 572 len 8192 data: 7 4 16/30 Write block 188 len 32768 data: 8 6 6 7 7 7 4 4 17/30 Flush 18/30 Write block 539 len 24576 data: 7 5 5 6 6 6 19/30 Read block 159 len 28672 20/30 Write block 53 len 40960 data: 7 5 6 5 5 6 5 6 7 6 21/30 Read block 97 len 20480 22/30 Read block 553 len 28672 23/30 Write block 437 len 12288 data: 3 2 2 24/30 Read block 252 len 28672 25/30 Write block 580 len 36864 data: 3 3 4 5 5 5 6 7 6 26/30 Flush 27/30 Write block 364 len 12288 data: 8 8 8 28/30 Read block 481 len 32768 29/30 Read block 12 len 12288 30/30 Write block 437 len 20480 data: 4 3 3 2 2 ---------------------------------------------------------------- Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Flush 0 Done Sent Done false 1001 Acked Write 7 Done Sent Done false 1002 Acked Read 8 Done Sent Done false 1003 Acked Write 2 Done Sent Done false 1004 Acked Read 7 Done Sent Done false 1005 Acked Write 7 Done Sent Done false 1006 Acked Read 9 Done Sent Done false 1007 Acked Read 1 Done Sent Done false 1008 Acked Read 8 Done Sent Done false 1009 Acked Read 10 Done Sent Done false 1010 Acked Read 6 Done Sent Done false 1011 Acked Read 4 Done Sent Done false 1012 Acked Write 1 Done Sent Done false 1013 Acked Read 3 Done Sent Done false 1014 Acked Write 2 Done Sent Done false 1015 Acked Write 8 Done Sent Done false 1016 Acked Flush 0 Done Sent Done false 1017 Acked Write 6 Done Sent Done false 1018 Acked Read 7 Done Sent Done false 1019 Acked Write 10 Done Sent Done false 1020 Acked Read 5 Done Sent Done false 1021 Acked Read 7 Done Sent Done false 1022 Acked Write 3 Done Sent Done false 1023 Acked Read 7 Done Sent Done false 1024 Acked Write 9 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 3 Done Sent Done false 1027 Acked Read 8 Done Sent Done false 1028 Acked Read 3 Done Sent Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 30 1 32 Done 29 0 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) None Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:21:41.186873775Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1376} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:41.187085935Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1376,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 68 68 68 F F F 1 020-039 56 56 56 80 80 80 F F F 2 040-059 60 58 60 88 84 88 F F F <--- 3 060-079 60 50 60 88 69 88 F F F <--- 4 080-099 47 47 47 64 64 64 F F F 5 100-119 56 56 56 82 82 82 F F F 6 120-139 58 58 58 84 84 84 F F F 7 140-159 50 50 50 69 69 69 F F F 8 160-179 54 54 54 77 77 77 F F F 9 180-199 60 58 60 87 84 87 F F F <--- 10 200-219 56 56 56 79 79 79 F F F 11 220-239 56 56 56 81 81 81 F F F 12 240-259 60 58 60 87 84 87 F F F <--- 13 260-279 60 57 60 87 83 87 F F F <--- 14 280-299 55 55 55 78 78 78 F F F 15 300-319 55 55 55 78 78 78 F F F 16 320-339 54 54 54 77 77 77 F F F 17 340-359 60 54 60 87 77 87 F F F <--- 18 360-379 60 58 60 87 84 87 T F T <--- 19 380-399 52 52 52 73 73 73 F F F 20 400-419 56 56 56 82 82 82 F F F 21 420-439 60 58 60 88 84 88 T F T <--- 22 440-459 58 58 58 84 84 84 T F T <--- 23 460-479 54 54 54 77 77 77 F F F 24 480-499 60 59 60 87 85 87 F F F <--- 25 500-519 56 56 56 80 80 80 F F F 26 520-539 60 54 60 88 76 88 F F F <--- 27 540-559 60 50 60 88 69 88 F F F <--- 28 560-579 60 58 60 87 84 87 F F F <--- 29 580-599 60 59 60 88 85 88 F F F <--- Max gen: 60, Max flush: 88 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 30 Verifying data now No change after verify Loop: 30 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 68 68 68 F F F 1 020-039 56 56 56 80 80 80 F F F 2 040-059 60 60 60 88 88 88 F F F 3 060-079 60 60 60 88 88 88 F F F 4 080-099 47 47 47 64 64 64 F F F 5 100-119 56 56 56 82 82 82 F F F 6 120-139 58 58 58 84 84 84 F F F 7 140-159 50 50 50 69 69 69 F F F 8 160-179 54 54 54 77 77 77 F F F 9 180-199 60 60 60 87 87 87 F F F 10 200-219 56 56 56 79 79 79 F F F 11 220-239 56 56 56 81 81 81 F F F 12 240-259 60 60 60 87 87 87 F F F 13 260-279 60 60 60 87 87 87 F F F 14 280-299 55 55 55 78 78 78 F F F 15 300-319 55 55 55 78 78 78 F F F 16 320-339 54 54 54 77 77 77 F F F 17 340-359 60 60 60 87 87 87 F F F 18 360-379 61 61 61 89 89 89 F F F 19 380-399 52 52 52 73 73 73 F F F 20 400-419 56 56 56 82 82 82 F F F 21 420-439 61 61 61 89 89 89 F F F 22 440-459 61 61 61 89 89 89 F F F 23 460-479 54 54 54 77 77 77 F F F 24 480-499 60 60 60 87 87 87 F F F 25 500-519 56 56 56 80 80 80 F F F 26 520-539 60 60 60 88 88 88 F F F 27 540-559 60 60 60 88 88 88 F F F 28 560-579 60 60 60 87 87 87 F F F 29 580-599 60 60 60 88 88 88 F F F Max gen: 61, Max flush: 89 Begin loop 31 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:45.436578088Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1384,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:45.44466039Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1384,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:45.44494929Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1384} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:45.4449779Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1384} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:45.44498673Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1384} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:45.44499863Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1384} {"msg":"Crucible 5d6a1a06-8a5c-459e-b960-7daf5fe64fa1 has session id: c3a12c8d-1b07-48bd-af1a-68fd827d7112","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:45.445366841Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1384,"session_id":"c3a12c8d-1b07-48bd-af1a-68fd827d7112"} {"msg":"Upstairs opts: Upstairs UUID: 5d6a1a06-8a5c-459e-b960-7daf5fe64fa1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:45.445396211Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1384,"session_id":"c3a12c8d-1b07-48bd-af1a-68fd827d7112"} {"msg":"Crucible stats registered with UUID: 5d6a1a06-8a5c-459e-b960-7daf5fe64fa1","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:45.445407361Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1384,"session_id":"c3a12c8d-1b07-48bd-af1a-68fd827d7112"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:21:45.445416941Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1384,"session_id":"c3a12c8d-1b07-48bd-af1a-68fd827d7112"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:45.445549491Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1384,"":"io task","client":"1","":"downstairs","session_id":"c3a12c8d-1b07-48bd-af1a-68fd827d7112"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:45.445567371Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1384,"":"io task","client":"0","":"downstairs","session_id":"c3a12c8d-1b07-48bd-af1a-68fd827d7112"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:45.445577641Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1384,"":"io task","client":"2","":"downstairs","session_id":"c3a12c8d-1b07-48bd-af1a-68fd827d7112"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:45.445734872Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1384,"task":"crutest"} {"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:45.445751752Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1384} {"msg":"5d6a1a06-8a5c-459e-b960-7daf5fe64fa1 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:45.445765042Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1384,"session_id":"c3a12c8d-1b07-48bd-af1a-68fd827d7112"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:45.445773822Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1384,"client":"0","":"downstairs","session_id":"c3a12c8d-1b07-48bd-af1a-68fd827d7112"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:45.445783762Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1384,"client":"1","":"downstairs","session_id":"c3a12c8d-1b07-48bd-af1a-68fd827d7112"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:45.445793382Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1384,"client":"2","":"downstairs","session_id":"c3a12c8d-1b07-48bd-af1a-68fd827d7112"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:45.445802972Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1384,"":"io task","client":"2","":"downstairs","session_id":"c3a12c8d-1b07-48bd-af1a-68fd827d7112"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:45.445813082Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1384,"":"io task","client":"0","":"downstairs","session_id":"c3a12c8d-1b07-48bd-af1a-68fd827d7112"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:45.445833282Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1384,"":"io task","client":"1","":"downstairs","session_id":"c3a12c8d-1b07-48bd-af1a-68fd827d7112"} {"msg":"ds_connection connected from Ok(127.0.0.1:48007)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:45.445893842Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1384,"":"io task","client":"2","":"downstairs","session_id":"c3a12c8d-1b07-48bd-af1a-68fd827d7112"} {"msg":"ds_connection connected from Ok(127.0.0.1:48912)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:45.445919202Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1384,"":"io task","client":"1","":"downstairs","session_id":"c3a12c8d-1b07-48bd-af1a-68fd827d7112"} {"msg":"ds_connection connected from Ok(127.0.0.1:50267)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:45.445930612Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1384,"":"io task","client":"0","":"downstairs","session_id":"c3a12c8d-1b07-48bd-af1a-68fd827d7112"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:45.446596873Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1384,"client":"1","":"downstairs","session_id":"c3a12c8d-1b07-48bd-af1a-68fd827d7112"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:45.446625793Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1384,"client":"2","":"downstairs","session_id":"c3a12c8d-1b07-48bd-af1a-68fd827d7112"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:45.446637543Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1384,"client":"0","":"downstairs","session_id":"c3a12c8d-1b07-48bd-af1a-68fd827d7112"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:45.446924373Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1384,"session_id":"c3a12c8d-1b07-48bd-af1a-68fd827d7112"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:45.446936933Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1384,"session_id":"c3a12c8d-1b07-48bd-af1a-68fd827d7112"} {"msg":"[0]R flush_numbers[0..12]: [68, 80, 88, 88, 64, 82, 84, 69, 77, 87, 79, 81]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:45.446974113Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1384,"":"downstairs","":"downstairs","session_id":"c3a12c8d-1b07-48bd-af1a-68fd827d7112"} {"msg":"[0]R generation[0..12]: [49, 56, 60, 60, 47, 56, 58, 50, 54, 60, 56, 56]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:45.446985263Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1384,"":"downstairs","":"downstairs","session_id":"c3a12c8d-1b07-48bd-af1a-68fd827d7112"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:45.446996413Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1384,"":"downstairs","":"downstairs","session_id":"c3a12c8d-1b07-48bd-af1a-68fd827d7112"} {"msg":"[1]R flush_numbers[0..12]: [68, 80, 88, 88, 64, 82, 84, 69, 77, 87, 79, 81]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:45.447006863Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1384,"":"downstairs","":"downstairs","session_id":"c3a12c8d-1b07-48bd-af1a-68fd827d7112"} {"msg":"[1]R generation[0..12]: [49, 56, 60, 60, 47, 56, 58, 50, 54, 60, 56, 56]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:45.447017553Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1384,"":"downstairs","":"downstairs","session_id":"c3a12c8d-1b07-48bd-af1a-68fd827d7112"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:45.447028714Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1384,"":"downstairs","":"downstairs","session_id":"c3a12c8d-1b07-48bd-af1a-68fd827d7112"} {"msg":"[2]R flush_numbers[0..12]: [68, 80, 88, 88, 64, 82, 84, 69, 77, 87, 79, 81]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:45.447039234Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1384,"":"downstairs","":"downstairs","session_id":"c3a12c8d-1b07-48bd-af1a-68fd827d7112"} {"msg":"[2]R generation[0..12]: [49, 56, 60, 60, 47, 56, 58, 50, 54, 60, 56, 56]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:45.447049794Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1384,"":"downstairs","":"downstairs","session_id":"c3a12c8d-1b07-48bd-af1a-68fd827d7112"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:45.447060724Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1384,"":"downstairs","":"downstairs","session_id":"c3a12c8d-1b07-48bd-af1a-68fd827d7112"} {"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:45.447070404Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1384,"":"downstairs","":"downstairs","session_id":"c3a12c8d-1b07-48bd-af1a-68fd827d7112"} {"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:45.447080104Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1384,"":"downstairs","":"downstairs","session_id":"c3a12c8d-1b07-48bd-af1a-68fd827d7112"} {"msg":"Next flush: 90","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:45.447089404Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1384,"":"downstairs","":"downstairs","session_id":"c3a12c8d-1b07-48bd-af1a-68fd827d7112"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:45.447098954Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1384,"":"downstairs","":"downstairs","session_id":"c3a12c8d-1b07-48bd-af1a-68fd827d7112"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:45.447109084Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1384,"session_id":"c3a12c8d-1b07-48bd-af1a-68fd827d7112"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:45.447119794Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1384,"client":"0","":"downstairs","session_id":"c3a12c8d-1b07-48bd-af1a-68fd827d7112"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:45.447129794Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1384,"client":"1","":"downstairs","session_id":"c3a12c8d-1b07-48bd-af1a-68fd827d7112"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:45.447139764Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1384,"client":"2","":"downstairs","session_id":"c3a12c8d-1b07-48bd-af1a-68fd827d7112"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:45.447149844Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1384,"session_id":"c3a12c8d-1b07-48bd-af1a-68fd827d7112"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:45.447159054Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1384,"session_id":"c3a12c8d-1b07-48bd-af1a-68fd827d7112"} {"msg":"5d6a1a06-8a5c-459e-b960-7daf5fe64fa1 is now active with session: c3a12c8d-1b07-48bd-af1a-68fd827d7112","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:45.447169094Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1384,"session_id":"c3a12c8d-1b07-48bd-af1a-68fd827d7112"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:45.447177894Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1384,"session_id":"c3a12c8d-1b07-48bd-af1a-68fd827d7112"} {"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:45.447187104Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1384} {"msg":"Activated sub_volume 5d6a1a06-8a5c-459e-b960-7daf5fe64fa1","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:45.447195124Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1384} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:45.447203294Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1384,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:45.447518774Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1384,"task":"crutest"} 01/30 Write block 403 len 32768 data: 2 4 4 4 5 6 5 5 02/30 Write block 394 len 12288 data: 7 6 5 03/30 Read block 580 len 20480 04/30 Write block 498 len 36864 data: 4 4 3 4 6 4 4 6 7 05/30 Write block 363 len 32768 data: 7 9 9 9 6 6 6 6 06/30 Flush 07/30 Write block 37 len 8192 data: 4 5 08/30 Write block 26 len 24576 data: 6 6 6 6 7 7 09/30 Read block 261 len 28672 10/30 Write block 531 len 4096 data: 6 11/30 Read block 436 len 32768 12/30 Write block 577 len 8192 data: 6 6 13/30 Flush 14/30 Write block 487 len 8192 data: 7 6 15/30 Read block 590 len 4096 16/30 Write block 449 len 40960 data: 5 4 3 3 3 4 5 6 6 8 17/30 Read block 254 len 24576 18/30 Read block 290 len 20480 19/30 Write block 352 len 16384 data: 4 5 6 5 20/30 Write block 310 len 24576 data: 5 4 4 4 4 3 21/30 Read block 346 len 8192 22/30 Read block 74 len 32768 23/30 Write block 27 len 8192 data: 7 7 24/30 Flush 25/30 Write block 499 len 28672 data: 5 4 5 7 5 5 7 26/30 Read block 150 len 32768 27/30 Write block 30 len 12288 data: 8 8 5 28/30 Read block 465 len 32768 29/30 Write block 584 len 24576 data: 6 6 7 8 7 5 30/30 Write block 106 len 40960 data: 8 7 7 7 8 7 6 6 5 4 ---------------------------------------------------------------- Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 2 Done Done Sent false 1007 Acked Write 6 Done Done Sent false 1008 Acked Read 7 Done Done Sent false 1009 Acked Write 1 Done Done Sent false 1010 Acked Read 8 Done Done Sent false 1011 Acked Write 2 Done Done Sent false 1012 Acked Flush 0 Done Done Sent false 1013 Acked Write 2 Done Done Sent false 1014 Acked Read 1 Done Done Sent false 1015 Acked Write 10 Done Done Sent false 1016 Acked Read 6 Done Done Sent false 1017 Acked Read 5 Done Done Sent false 1018 Acked Write 4 Done Done Sent false 1019 Acked Write 6 Done Done Sent false 1020 Acked Read 2 Done Done Sent false 1021 Acked Read 8 Done Done Sent false 1022 Acked Write 2 Done Done Sent false 1023 Acked Flush 0 Done Done Sent false 1024 Acked Write 7 Done Done Sent false 1025 Acked Read 8 Done Done Sent false 1026 Acked Write 3 Done Done Sent false 1027 Acked Read 8 Done Done Sent false 1028 Acked Write 6 Done Sent Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 24 27 Done 23 22 0 45 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1005)) Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:21:45.455991927Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1384} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:45.456174717Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1384,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 68 68 68 F F F 1 020-039 62 62 56 92 92 80 T T F <--- 2 040-059 60 60 60 88 88 88 F F F 3 060-079 60 60 60 88 88 88 F F F 4 080-099 47 47 47 64 64 64 F F F 5 100-119 56 56 56 82 82 82 T T F <--- 6 120-139 58 58 58 84 84 84 F F F 7 140-159 50 50 50 69 69 69 F F F 8 160-179 54 54 54 77 77 77 F F F 9 180-199 60 60 60 87 87 87 F F F 10 200-219 56 56 56 79 79 79 F F F 11 220-239 56 56 56 81 81 81 F F F 12 240-259 60 60 60 87 87 87 F F F 13 260-279 60 60 60 87 87 87 F F F 14 280-299 55 55 55 78 78 78 F F F 15 300-319 62 62 55 92 92 78 F F F <--- 16 320-339 54 54 54 77 77 77 F F F 17 340-359 62 62 60 92 92 87 F F F <--- 18 360-379 62 62 62 90 90 90 F F F 19 380-399 62 62 62 90 90 90 F F F 20 400-419 62 62 62 90 90 90 F F F 21 420-439 61 61 61 89 89 89 F F F 22 440-459 62 62 61 92 92 89 F F F <--- 23 460-479 54 54 54 77 77 77 F F F 24 480-499 62 62 62 92 92 90 T T F <--- 25 500-519 62 62 62 90 90 90 T T F <--- 26 520-539 62 62 60 91 91 88 F F F <--- 27 540-559 60 60 60 88 88 88 F F F 28 560-579 62 62 60 91 91 87 F F F <--- 29 580-599 60 60 60 88 88 88 T T F <--- Max gen: 62, Max flush: 92 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 31 Verifying data now No change after verify Loop: 31 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 68 68 68 F F F 1 020-039 63 63 63 93 93 93 F F F 2 040-059 60 60 60 88 88 88 F F F 3 060-079 60 60 60 88 88 88 F F F 4 080-099 47 47 47 64 64 64 F F F 5 100-119 63 63 63 93 93 93 F F F 6 120-139 58 58 58 84 84 84 F F F 7 140-159 50 50 50 69 69 69 F F F 8 160-179 54 54 54 77 77 77 F F F 9 180-199 60 60 60 87 87 87 F F F 10 200-219 56 56 56 79 79 79 F F F 11 220-239 56 56 56 81 81 81 F F F 12 240-259 60 60 60 87 87 87 F F F 13 260-279 60 60 60 87 87 87 F F F 14 280-299 55 55 55 78 78 78 F F F 15 300-319 62 62 62 92 92 92 F F F 16 320-339 54 54 54 77 77 77 F F F 17 340-359 62 62 62 92 92 92 F F F 18 360-379 62 62 62 90 90 90 F F F 19 380-399 62 62 62 90 90 90 F F F 20 400-419 62 62 62 90 90 90 F F F 21 420-439 61 61 61 89 89 89 F F F 22 440-459 62 62 62 92 92 92 F F F 23 460-479 54 54 54 77 77 77 F F F 24 480-499 63 63 63 93 93 93 F F F 25 500-519 63 63 63 93 93 93 F F F 26 520-539 62 62 62 91 91 91 F F F 27 540-559 60 60 60 88 88 88 F F F 28 560-579 62 62 62 91 91 91 F F F 29 580-599 63 63 63 93 93 93 F F F Max gen: 63, Max flush: 93 Begin loop 32 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:49.703209076Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1392,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:21:49.710354367Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1392,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:49.710408387Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1392,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:49.717232757Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1392,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:49.717498528Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1392} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:49.717526528Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1392} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:49.717537398Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1392} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:49.717549098Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1392} {"msg":"Crucible 2b152265-319c-416a-819a-21003b53fab1 has session id: 85fb30d2-cbb7-4828-8241-439246589fe6","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:49.717911428Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1392,"session_id":"85fb30d2-cbb7-4828-8241-439246589fe6"} {"msg":"Upstairs opts: Upstairs UUID: 2b152265-319c-416a-819a-21003b53fab1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:49.717932968Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1392,"session_id":"85fb30d2-cbb7-4828-8241-439246589fe6"} {"msg":"Crucible stats registered with UUID: 2b152265-319c-416a-819a-21003b53fab1","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:49.717943138Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1392,"session_id":"85fb30d2-cbb7-4828-8241-439246589fe6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:21:49.717952468Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1392,"session_id":"85fb30d2-cbb7-4828-8241-439246589fe6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:49.718095269Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1392,"":"io task","client":"1","":"downstairs","session_id":"85fb30d2-cbb7-4828-8241-439246589fe6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:49.718113079Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1392,"":"io task","client":"0","":"downstairs","session_id":"85fb30d2-cbb7-4828-8241-439246589fe6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:49.718123119Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1392,"":"io task","client":"2","":"downstairs","session_id":"85fb30d2-cbb7-4828-8241-439246589fe6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:49.718278569Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1392,"task":"crutest"} {"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:49.718290229Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1392} {"msg":"2b152265-319c-416a-819a-21003b53fab1 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:49.718302379Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1392,"session_id":"85fb30d2-cbb7-4828-8241-439246589fe6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:49.718311229Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1392,"client":"0","":"downstairs","session_id":"85fb30d2-cbb7-4828-8241-439246589fe6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:49.718321019Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1392,"client":"1","":"downstairs","session_id":"85fb30d2-cbb7-4828-8241-439246589fe6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:49.718330659Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1392,"client":"2","":"downstairs","session_id":"85fb30d2-cbb7-4828-8241-439246589fe6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:49.718341989Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1392,"":"io task","client":"2","":"downstairs","session_id":"85fb30d2-cbb7-4828-8241-439246589fe6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:49.718352039Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1392,"":"io task","client":"0","":"downstairs","session_id":"85fb30d2-cbb7-4828-8241-439246589fe6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:49.718362129Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1392,"":"io task","client":"1","":"downstairs","session_id":"85fb30d2-cbb7-4828-8241-439246589fe6"} {"msg":"ds_connection connected from Ok(127.0.0.1:34046)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:49.718442169Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1392,"":"io task","client":"1","":"downstairs","session_id":"85fb30d2-cbb7-4828-8241-439246589fe6"} {"msg":"ds_connection connected from Ok(127.0.0.1:43769)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:49.718467559Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1392,"":"io task","client":"2","":"downstairs","session_id":"85fb30d2-cbb7-4828-8241-439246589fe6"} {"msg":"ds_connection connected from Ok(127.0.0.1:43080)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:49.718535239Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1392,"":"io task","client":"0","":"downstairs","session_id":"85fb30d2-cbb7-4828-8241-439246589fe6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:49.71913506Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1392,"client":"1","":"downstairs","session_id":"85fb30d2-cbb7-4828-8241-439246589fe6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:49.71918599Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1392,"client":"0","":"downstairs","session_id":"85fb30d2-cbb7-4828-8241-439246589fe6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:49.71920488Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1392,"client":"2","":"downstairs","session_id":"85fb30d2-cbb7-4828-8241-439246589fe6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:49.719445591Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1392,"session_id":"85fb30d2-cbb7-4828-8241-439246589fe6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:49.719554551Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1392,"session_id":"85fb30d2-cbb7-4828-8241-439246589fe6"} {"msg":"[0]R flush_numbers[0..12]: [68, 93, 88, 88, 64, 93, 84, 69, 77, 87, 79, 81]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:49.719589831Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1392,"":"downstairs","":"downstairs","session_id":"85fb30d2-cbb7-4828-8241-439246589fe6"} {"msg":"[0]R generation[0..12]: [49, 63, 60, 60, 47, 63, 58, 50, 54, 60, 56, 56]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:49.719601111Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1392,"":"downstairs","":"downstairs","session_id":"85fb30d2-cbb7-4828-8241-439246589fe6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:49.719612231Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1392,"":"downstairs","":"downstairs","session_id":"85fb30d2-cbb7-4828-8241-439246589fe6"} {"msg":"[1]R flush_numbers[0..12]: [68, 93, 88, 88, 64, 93, 84, 69, 77, 87, 79, 81]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:49.719622911Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1392,"":"downstairs","":"downstairs","session_id":"85fb30d2-cbb7-4828-8241-439246589fe6"} {"msg":"[1]R generation[0..12]: [49, 63, 60, 60, 47, 63, 58, 50, 54, 60, 56, 56]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:49.719635621Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1392,"":"downstairs","":"downstairs","session_id":"85fb30d2-cbb7-4828-8241-439246589fe6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:49.719646791Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1392,"":"downstairs","":"downstairs","session_id":"85fb30d2-cbb7-4828-8241-439246589fe6"} {"msg":"[2]R flush_numbers[0..12]: [68, 93, 88, 88, 64, 93, 84, 69, 77, 87, 79, 81]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:49.719657521Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1392,"":"downstairs","":"downstairs","session_id":"85fb30d2-cbb7-4828-8241-439246589fe6"} {"msg":"[2]R generation[0..12]: [49, 63, 60, 60, 47, 63, 58, 50, 54, 60, 56, 56]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:49.719668031Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1392,"":"downstairs","":"downstairs","session_id":"85fb30d2-cbb7-4828-8241-439246589fe6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:49.719679111Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1392,"":"downstairs","":"downstairs","session_id":"85fb30d2-cbb7-4828-8241-439246589fe6"} {"msg":"Max found gen is 64","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:49.719688571Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1392,"":"downstairs","":"downstairs","session_id":"85fb30d2-cbb7-4828-8241-439246589fe6"} {"msg":"Generation requested: 64 >= found:64","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:49.719698381Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1392,"":"downstairs","":"downstairs","session_id":"85fb30d2-cbb7-4828-8241-439246589fe6"} {"msg":"Next flush: 94","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:49.719707651Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1392,"":"downstairs","":"downstairs","session_id":"85fb30d2-cbb7-4828-8241-439246589fe6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:49.719717051Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1392,"":"downstairs","":"downstairs","session_id":"85fb30d2-cbb7-4828-8241-439246589fe6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:49.719727211Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1392,"session_id":"85fb30d2-cbb7-4828-8241-439246589fe6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:49.719736151Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1392,"client":"0","":"downstairs","session_id":"85fb30d2-cbb7-4828-8241-439246589fe6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:49.719746161Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1392,"client":"1","":"downstairs","session_id":"85fb30d2-cbb7-4828-8241-439246589fe6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:49.719756161Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1392,"client":"2","":"downstairs","session_id":"85fb30d2-cbb7-4828-8241-439246589fe6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:49.719766291Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1392,"session_id":"85fb30d2-cbb7-4828-8241-439246589fe6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:49.719775561Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1392,"session_id":"85fb30d2-cbb7-4828-8241-439246589fe6"} {"msg":"2b152265-319c-416a-819a-21003b53fab1 is now active with session: 85fb30d2-cbb7-4828-8241-439246589fe6","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:49.719786721Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1392,"session_id":"85fb30d2-cbb7-4828-8241-439246589fe6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:49.719795461Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1392,"session_id":"85fb30d2-cbb7-4828-8241-439246589fe6"} {"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:49.719804841Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1392} {"msg":"Activated sub_volume 2b152265-319c-416a-819a-21003b53fab1","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:49.719812911Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1392} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:49.719821071Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1392,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:49.720151632Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1392,"task":"crutest"} 01/30 Write block 553 len 4096 data: 3 02/30 Read block 584 len 12288 03/30 Write block 455 len 20480 data: 6 7 7 9 7 04/30 Read block 107 len 28672 05/30 Read block 575 len 8192 06/30 Read block 363 len 40960 07/30 Read block 426 len 8192 08/30 Read block 251 len 4096 09/30 Read block 182 len 8192 10/30 Write block 463 len 16384 data: 6 5 4 5 11/30 Write block 234 len 40960 data: 5 5 7 9 8 10 8 7 7 7 12/30 Write block 375 len 20480 data: 9 10 8 8 7 13/30 Read block 458 len 40960 14/30 Flush 15/30 Write block 485 len 40960 data: 7 7 8 7 9 7 5 8 8 6 16/30 Read block 142 len 24576 17/30 Read block 366 len 4096 18/30 Write block 423 len 32768 data: 6 6 5 4 4 4 4 3 19/30 Write block 343 len 24576 data: 5 5 6 7 8 9 20/30 Read block 517 len 20480 21/30 Read block 75 len 28672 22/30 Write block 572 len 32768 data: 8 5 4 6 7 7 7 4 23/30 Flush 24/30 Write block 200 len 28672 data: 6 5 6 8 8 8 6 25/30 Read block 196 len 28672 26/30 Write block 525 len 28672 data: 9 8 10 11 10 7 7 27/30 Read block 471 len 36864 28/30 Read block 268 len 16384 29/30 Write block 513 len 28672 data: 5 4 2 2 3 4 5 30/30 Write block 203 len 32768 data: 9 9 9 7 7 7 8 9 ---------------------------------------------------------------- Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Sent Done Done false 1001 Acked Read 3 Sent Done Done false 1002 Acked Write 5 Sent Done Done false 1003 Acked Read 7 Sent Done Done false 1004 Acked Read 2 Sent Done Done false 1005 Acked Read 10 Sent Done Done false 1006 Acked Read 2 Sent Done Done false 1007 Acked Read 1 Sent Done Done false 1008 Acked Read 2 Sent Done Done false 1009 Acked Write 4 Sent Done Done false 1010 Acked Write 10 Sent Done Done false 1011 Acked Write 5 Sent Done Done false 1012 Acked Read 10 Sent Done Done false 1013 Acked Flush 0 Sent Done Done false 1014 Acked Write 10 Sent Done Done false 1015 Acked Read 6 Sent Done Done false 1016 Acked Read 1 Sent Done Done false 1017 Acked Write 8 Sent Done Done false 1018 Acked Write 6 Sent Done Done false 1019 Acked Read 5 Sent Done Done false 1020 Acked Read 7 Sent Done Done false 1021 Acked Write 8 Sent Done Done false 1022 Acked Flush 0 Sent Done Done false 1023 Acked Write 7 Sent Done Done false 1024 Acked Read 7 Sent Done Done false 1025 Acked Write 7 Sent Done Done false 1026 Acked Read 9 Sent Done Done false 1027 Acked Read 4 Sent Done Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 2 2 34 Done 0 28 28 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:21:49.728808225Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1392} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:49.728974075Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1392,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 68 68 68 F F F 1 020-039 63 63 63 93 93 93 F F F 2 040-059 60 60 60 88 88 88 F F F 3 060-079 60 60 60 88 88 88 F F F 4 080-099 47 47 47 64 64 64 F F F 5 100-119 63 63 63 93 93 93 F F F 6 120-139 58 58 58 84 84 84 F F F 7 140-159 50 50 50 69 69 69 F F F 8 160-179 54 54 54 77 77 77 F F F 9 180-199 60 60 60 87 87 87 F F F 10 200-219 56 56 56 79 79 79 F T T <--- 11 220-239 56 64 64 81 94 94 F F F <--- 12 240-259 60 64 64 87 94 94 F F F <--- 13 260-279 60 60 60 87 87 87 F F F 14 280-299 55 55 55 78 78 78 F F F 15 300-319 62 62 62 92 92 92 F F F 16 320-339 54 54 54 77 77 77 F F F 17 340-359 62 64 64 92 95 95 F F F <--- 18 360-379 62 64 64 90 94 94 F F F <--- 19 380-399 62 62 62 90 90 90 F F F 20 400-419 62 62 62 90 90 90 F F F 21 420-439 61 64 64 89 95 95 F F F <--- 22 440-459 62 64 64 92 94 94 F F F <--- 23 460-479 54 64 64 77 94 94 F F F <--- 24 480-499 63 64 64 93 95 95 F F F <--- 25 500-519 63 63 63 93 93 93 F T T <--- 26 520-539 62 62 62 91 91 91 F T T <--- 27 540-559 60 64 64 88 94 94 F F F <--- 28 560-579 62 64 64 91 95 95 F F F <--- 29 580-599 63 63 63 93 93 93 F F F Max gen: 64, Max flush: 95 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 32 Verifying data now No change after verify Loop: 32 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 68 68 68 F F F 1 020-039 63 63 63 93 93 93 F F F 2 040-059 60 60 60 88 88 88 F F F 3 060-079 60 60 60 88 88 88 F F F 4 080-099 47 47 47 64 64 64 F F F 5 100-119 63 63 63 93 93 93 F F F 6 120-139 58 58 58 84 84 84 F F F 7 140-159 50 50 50 69 69 69 F F F 8 160-179 54 54 54 77 77 77 F F F 9 180-199 60 60 60 87 87 87 F F F 10 200-219 65 65 65 96 96 96 F F F 11 220-239 64 64 64 94 94 94 F F F 12 240-259 64 64 64 94 94 94 F F F 13 260-279 60 60 60 87 87 87 F F F 14 280-299 55 55 55 78 78 78 F F F 15 300-319 62 62 62 92 92 92 F F F 16 320-339 54 54 54 77 77 77 F F F 17 340-359 64 64 64 95 95 95 F F F 18 360-379 64 64 64 94 94 94 F F F 19 380-399 62 62 62 90 90 90 F F F 20 400-419 62 62 62 90 90 90 F F F 21 420-439 64 64 64 95 95 95 F F F 22 440-459 64 64 64 94 94 94 F F F 23 460-479 64 64 64 94 94 94 F F F 24 480-499 64 64 64 95 95 95 F F F 25 500-519 65 65 65 96 96 96 F F F 26 520-539 65 65 65 96 96 96 F F F 27 540-559 64 64 64 94 94 94 F F F 28 560-579 64 64 64 95 95 95 F F F 29 580-599 63 63 63 93 93 93 F F F Max gen: 65, Max flush: 96 Begin loop 33 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:53.984057316Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1400,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:21:53.991241756Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1400,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:53.991297267Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1400,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:53.997997677Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1400,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:53.998266847Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1400} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:53.998294227Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1400} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:53.998302757Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1400} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:53.998314487Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1400} {"msg":"Crucible 5f0f81ff-673a-496c-9c7b-851f862a81d3 has session id: 16335438-ac47-4335-b91d-0d12edb03acf","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:53.998668968Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1400,"session_id":"16335438-ac47-4335-b91d-0d12edb03acf"} {"msg":"Upstairs opts: Upstairs UUID: 5f0f81ff-673a-496c-9c7b-851f862a81d3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:53.998698208Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1400,"session_id":"16335438-ac47-4335-b91d-0d12edb03acf"} {"msg":"Crucible stats registered with UUID: 5f0f81ff-673a-496c-9c7b-851f862a81d3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:53.998711628Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1400,"session_id":"16335438-ac47-4335-b91d-0d12edb03acf"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:21:53.998720918Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1400,"session_id":"16335438-ac47-4335-b91d-0d12edb03acf"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:53.998854018Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1400,"":"io task","client":"1","":"downstairs","session_id":"16335438-ac47-4335-b91d-0d12edb03acf"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:53.998873598Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1400,"":"io task","client":"0","":"downstairs","session_id":"16335438-ac47-4335-b91d-0d12edb03acf"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:53.998883738Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1400,"":"io task","client":"2","":"downstairs","session_id":"16335438-ac47-4335-b91d-0d12edb03acf"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:53.999038958Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1400,"task":"crutest"} {"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:53.999055938Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1400} {"msg":"5f0f81ff-673a-496c-9c7b-851f862a81d3 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:53.999088958Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1400,"session_id":"16335438-ac47-4335-b91d-0d12edb03acf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:53.999098558Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1400,"client":"0","":"downstairs","session_id":"16335438-ac47-4335-b91d-0d12edb03acf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:53.999108528Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1400,"client":"1","":"downstairs","session_id":"16335438-ac47-4335-b91d-0d12edb03acf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:53.999118378Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1400,"client":"2","":"downstairs","session_id":"16335438-ac47-4335-b91d-0d12edb03acf"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:53.999128088Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1400,"":"io task","client":"2","":"downstairs","session_id":"16335438-ac47-4335-b91d-0d12edb03acf"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:53.999138218Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1400,"":"io task","client":"0","":"downstairs","session_id":"16335438-ac47-4335-b91d-0d12edb03acf"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:53.999148338Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1400,"":"io task","client":"1","":"downstairs","session_id":"16335438-ac47-4335-b91d-0d12edb03acf"} {"msg":"ds_connection connected from Ok(127.0.0.1:40588)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:53.999186748Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1400,"":"io task","client":"2","":"downstairs","session_id":"16335438-ac47-4335-b91d-0d12edb03acf"} {"msg":"ds_connection connected from Ok(127.0.0.1:36444)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:53.999217089Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1400,"":"io task","client":"0","":"downstairs","session_id":"16335438-ac47-4335-b91d-0d12edb03acf"} {"msg":"ds_connection connected from Ok(127.0.0.1:64534)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:53.999240279Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1400,"":"io task","client":"1","":"downstairs","session_id":"16335438-ac47-4335-b91d-0d12edb03acf"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:53.99988718Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1400,"client":"2","":"downstairs","session_id":"16335438-ac47-4335-b91d-0d12edb03acf"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:53.99993232Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1400,"client":"1","":"downstairs","session_id":"16335438-ac47-4335-b91d-0d12edb03acf"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:53.99994542Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1400,"client":"0","":"downstairs","session_id":"16335438-ac47-4335-b91d-0d12edb03acf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:54.00019814Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1400,"session_id":"16335438-ac47-4335-b91d-0d12edb03acf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:54.00026658Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1400,"session_id":"16335438-ac47-4335-b91d-0d12edb03acf"} {"msg":"[0]R flush_numbers[0..12]: [68, 93, 88, 88, 64, 93, 84, 69, 77, 87, 96, 94]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:54.00029112Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1400,"":"downstairs","":"downstairs","session_id":"16335438-ac47-4335-b91d-0d12edb03acf"} {"msg":"[0]R generation[0..12]: [49, 63, 60, 60, 47, 63, 58, 50, 54, 60, 65, 64]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:54.00030311Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1400,"":"downstairs","":"downstairs","session_id":"16335438-ac47-4335-b91d-0d12edb03acf"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:54.00031469Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1400,"":"downstairs","":"downstairs","session_id":"16335438-ac47-4335-b91d-0d12edb03acf"} {"msg":"[1]R flush_numbers[0..12]: [68, 93, 88, 88, 64, 93, 84, 69, 77, 87, 96, 94]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:54.00032544Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1400,"":"downstairs","":"downstairs","session_id":"16335438-ac47-4335-b91d-0d12edb03acf"} {"msg":"[1]R generation[0..12]: [49, 63, 60, 60, 47, 63, 58, 50, 54, 60, 65, 64]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:54.00033614Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1400,"":"downstairs","":"downstairs","session_id":"16335438-ac47-4335-b91d-0d12edb03acf"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:54.0003472Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1400,"":"downstairs","":"downstairs","session_id":"16335438-ac47-4335-b91d-0d12edb03acf"} {"msg":"[2]R flush_numbers[0..12]: [68, 93, 88, 88, 64, 93, 84, 69, 77, 87, 96, 94]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:54.00035831Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1400,"":"downstairs","":"downstairs","session_id":"16335438-ac47-4335-b91d-0d12edb03acf"} {"msg":"[2]R generation[0..12]: [49, 63, 60, 60, 47, 63, 58, 50, 54, 60, 65, 64]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:54.00036891Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1400,"":"downstairs","":"downstairs","session_id":"16335438-ac47-4335-b91d-0d12edb03acf"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:54.00038212Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1400,"":"downstairs","":"downstairs","session_id":"16335438-ac47-4335-b91d-0d12edb03acf"} {"msg":"Max found gen is 66","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:54.00039179Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1400,"":"downstairs","":"downstairs","session_id":"16335438-ac47-4335-b91d-0d12edb03acf"} {"msg":"Generation requested: 66 >= found:66","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:54.00040163Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1400,"":"downstairs","":"downstairs","session_id":"16335438-ac47-4335-b91d-0d12edb03acf"} {"msg":"Next flush: 97","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:54.00041103Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1400,"":"downstairs","":"downstairs","session_id":"16335438-ac47-4335-b91d-0d12edb03acf"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:54.00042057Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1400,"":"downstairs","":"downstairs","session_id":"16335438-ac47-4335-b91d-0d12edb03acf"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:54.00043044Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1400,"session_id":"16335438-ac47-4335-b91d-0d12edb03acf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:54.00043944Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1400,"client":"0","":"downstairs","session_id":"16335438-ac47-4335-b91d-0d12edb03acf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:54.00044922Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1400,"client":"1","":"downstairs","session_id":"16335438-ac47-4335-b91d-0d12edb03acf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:54.00045892Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1400,"client":"2","":"downstairs","session_id":"16335438-ac47-4335-b91d-0d12edb03acf"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:54.00046877Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1400,"session_id":"16335438-ac47-4335-b91d-0d12edb03acf"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:54.00047797Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1400,"session_id":"16335438-ac47-4335-b91d-0d12edb03acf"} {"msg":"5f0f81ff-673a-496c-9c7b-851f862a81d3 is now active with session: 16335438-ac47-4335-b91d-0d12edb03acf","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:54.00048806Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1400,"session_id":"16335438-ac47-4335-b91d-0d12edb03acf"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:54.00049695Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1400,"session_id":"16335438-ac47-4335-b91d-0d12edb03acf"} {"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:54.00050645Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1400} {"msg":"Activated sub_volume 5f0f81ff-673a-496c-9c7b-851f862a81d3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:54.00051471Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1400} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:54.000522941Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1400,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:54.000823811Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1400,"task":"crutest"} 01/30 Write block 274 len 36864 data: 9 10 11 11 6 4 5 3 3 02/30 Read block 192 len 40960 03/30 Read block 583 len 36864 04/30 Read block 542 len 12288 05/30 Write block 241 len 28672 data: 8 8 8 7 9 8 8 06/30 Read block 244 len 20480 07/30 Read block 494 len 32768 08/30 Read block 155 len 8192 09/30 Write block 138 len 12288 data: 7 10 9 10/30 Write block 416 len 24576 data: 5 5 5 4 4 6 11/30 Write block 144 len 36864 data: 6 5 5 6 6 8 7 7 7 12/30 Write block 234 len 40960 data: 6 6 8 10 9 11 9 9 9 9 13/30 Read block 124 len 36864 14/30 Write block 286 len 16384 data: 3 4 4 5 15/30 Read block 144 len 20480 16/30 Write block 167 len 24576 data: 4 4 5 5 5 5 17/30 Write block 192 len 4096 data: 8 18/30 Read block 491 len 8192 19/30 Write block 224 len 36864 data: 2 4 4 3 6 4 4 3 4 20/30 Flush 21/30 Write block 476 len 16384 data: 5 5 5 5 22/30 Write block 158 len 12288 data: 6 5 6 23/30 Read block 365 len 4096 24/30 Read block 127 len 20480 25/30 Write block 392 len 16384 data: 8 6 8 7 26/30 Write block 164 len 20480 data: 3 4 4 5 5 27/30 Write block 298 len 16384 data: 8 9 7 10 28/30 Write block 433 len 4096 data: 3 29/30 Write block 192 len 40960 data: 9 8 5 5 5 6 6 5 7 6 30/30 Read block 475 len 8192 ---------------------------------------------------------------- Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Read 10 Done Done Done false 1002 Acked Read 9 Done Done Done false 1003 Acked Read 3 Sent Done Done false 1004 Acked Write 7 Sent Done Done false 1005 Acked Read 5 Sent Done Done false 1006 Acked Read 8 Sent Done Done false 1007 Acked Read 2 Sent Done Done false 1008 Acked Write 3 Sent Done Done false 1009 Acked Write 6 Sent Done Done false 1010 Acked Write 9 Sent Done Done false 1011 Acked Write 10 Sent Done Done false 1012 Acked Read 9 Sent Done Done false 1013 Acked Write 4 Sent Done Done false 1014 Acked Read 5 Sent Done Done false 1015 Acked Write 6 Sent Done Done false 1016 Acked Write 1 Sent Done Done false 1017 Acked Read 2 Sent Done Done false 1018 Acked Write 9 Sent Done Done false 1019 Acked Flush 0 Sent Done Done false 1020 Acked Write 4 Sent Done Done false 1021 Acked Write 3 Sent Done Done false 1022 Acked Read 1 Sent Done Done false 1023 Acked Read 5 Sent Done Done false 1024 Acked Write 4 Sent Done Done false 1025 Acked Write 5 Sent Done Done false 1026 Acked Write 4 Sent Done Done false 1027 Acked Write 1 Sent Done Done false 1028 Acked Write 10 Sent Done Sent false 1029 Acked Read 2 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 27 0 2 29 Done 3 30 28 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:21:54.009503284Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1400} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:54.009666954Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1400,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 68 68 68 F F F 1 020-039 63 63 63 93 93 93 F F F 2 040-059 60 60 60 88 88 88 F F F 3 060-079 60 60 60 88 88 88 F F F 4 080-099 47 47 47 64 64 64 F F F 5 100-119 63 63 63 93 93 93 F F F 6 120-139 58 66 66 84 97 97 F F F <--- 7 140-159 50 66 66 69 97 97 F T T <--- 8 160-179 54 66 66 77 97 97 F T T <--- 9 180-199 60 66 66 87 97 97 F T T <--- 10 200-219 65 65 65 96 96 96 F T T <--- 11 220-239 64 66 66 94 97 97 F F F <--- 12 240-259 64 66 66 94 97 97 F F F <--- 13 260-279 60 66 66 87 97 97 T F F <--- 14 280-299 55 66 66 78 97 97 T T T <--- 15 300-319 62 62 62 92 92 92 F T T <--- 16 320-339 54 54 54 77 77 77 F F F 17 340-359 64 64 64 95 95 95 F F F 18 360-379 64 64 64 94 94 94 F F F 19 380-399 62 62 62 90 90 90 F T T <--- 20 400-419 62 66 66 90 97 97 F F F <--- 21 420-439 64 66 66 95 97 97 F T T <--- 22 440-459 64 64 64 94 94 94 F F F 23 460-479 64 64 64 94 94 94 F T T <--- 24 480-499 64 64 64 95 95 95 F F F 25 500-519 65 65 65 96 96 96 F F F 26 520-539 65 65 65 96 96 96 F F F 27 540-559 64 64 64 94 94 94 F F F 28 560-579 64 64 64 95 95 95 F F F 29 580-599 63 63 63 93 93 93 F F F Max gen: 66, Max flush: 97 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 33 Verifying data now No change after verify Loop: 33 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 68 68 68 F F F 1 020-039 63 63 63 93 93 93 F F F 2 040-059 60 60 60 88 88 88 F F F 3 060-079 60 60 60 88 88 88 F F F 4 080-099 47 47 47 64 64 64 F F F 5 100-119 63 63 63 93 93 93 F F F 6 120-139 66 66 66 97 97 97 F F F 7 140-159 67 67 67 98 98 98 F F F 8 160-179 67 67 67 98 98 98 F F F 9 180-199 67 67 67 98 98 98 F F F 10 200-219 67 67 67 98 98 98 F F F 11 220-239 66 66 66 97 97 97 F F F 12 240-259 66 66 66 97 97 97 F F F 13 260-279 66 66 66 97 97 97 F F F 14 280-299 67 67 67 98 98 98 F F F 15 300-319 67 67 67 98 98 98 F F F 16 320-339 54 54 54 77 77 77 F F F 17 340-359 64 64 64 95 95 95 F F F 18 360-379 64 64 64 94 94 94 F F F 19 380-399 67 67 67 98 98 98 F F F 20 400-419 66 66 66 97 97 97 F F F 21 420-439 67 67 67 98 98 98 F F F 22 440-459 64 64 64 94 94 94 F F F 23 460-479 67 67 67 98 98 98 F F F 24 480-499 64 64 64 95 95 95 F F F 25 500-519 65 65 65 96 96 96 F F F 26 520-539 65 65 65 96 96 96 F F F 27 540-559 64 64 64 94 94 94 F F F 28 560-579 64 64 64 95 95 95 F F F 29 580-599 63 63 63 93 93 93 F F F Max gen: 67, Max flush: 98 Begin loop 34 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:58.270654684Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1408,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:58.279033487Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1408,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:58.279335777Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1408} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:58.279361347Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1408} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:58.279370167Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1408} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:58.279382017Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1408} {"msg":"Crucible d7cbe96d-b1d4-4cde-b1a9-23b579513138 has session id: 39b47c96-9b2f-4901-9866-ab4ad8a5098a","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:58.279739688Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1408,"session_id":"39b47c96-9b2f-4901-9866-ab4ad8a5098a"} {"msg":"Upstairs opts: Upstairs UUID: d7cbe96d-b1d4-4cde-b1a9-23b579513138, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:58.279768948Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1408,"session_id":"39b47c96-9b2f-4901-9866-ab4ad8a5098a"} {"msg":"Crucible stats registered with UUID: d7cbe96d-b1d4-4cde-b1a9-23b579513138","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:58.279780228Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1408,"session_id":"39b47c96-9b2f-4901-9866-ab4ad8a5098a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:21:58.279789588Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1408,"session_id":"39b47c96-9b2f-4901-9866-ab4ad8a5098a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:58.279922768Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1408,"":"io task","client":"2","":"downstairs","session_id":"39b47c96-9b2f-4901-9866-ab4ad8a5098a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:58.279942138Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1408,"":"io task","client":"0","":"downstairs","session_id":"39b47c96-9b2f-4901-9866-ab4ad8a5098a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:58.279952418Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1408,"":"io task","client":"1","":"downstairs","session_id":"39b47c96-9b2f-4901-9866-ab4ad8a5098a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:58.280105089Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1408,"task":"crutest"} {"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:58.280122079Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1408} {"msg":"d7cbe96d-b1d4-4cde-b1a9-23b579513138 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:58.280138119Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1408,"session_id":"39b47c96-9b2f-4901-9866-ab4ad8a5098a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:58.280146989Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1408,"client":"0","":"downstairs","session_id":"39b47c96-9b2f-4901-9866-ab4ad8a5098a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:58.280156959Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1408,"client":"1","":"downstairs","session_id":"39b47c96-9b2f-4901-9866-ab4ad8a5098a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:58.280166829Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1408,"client":"2","":"downstairs","session_id":"39b47c96-9b2f-4901-9866-ab4ad8a5098a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:58.280176299Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1408,"":"io task","client":"2","":"downstairs","session_id":"39b47c96-9b2f-4901-9866-ab4ad8a5098a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:58.280186459Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1408,"":"io task","client":"0","":"downstairs","session_id":"39b47c96-9b2f-4901-9866-ab4ad8a5098a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:58.280196499Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1408,"":"io task","client":"1","":"downstairs","session_id":"39b47c96-9b2f-4901-9866-ab4ad8a5098a"} {"msg":"ds_connection connected from Ok(127.0.0.1:59814)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:58.280235119Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1408,"":"io task","client":"2","":"downstairs","session_id":"39b47c96-9b2f-4901-9866-ab4ad8a5098a"} {"msg":"ds_connection connected from Ok(127.0.0.1:56910)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:58.280286899Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1408,"":"io task","client":"0","":"downstairs","session_id":"39b47c96-9b2f-4901-9866-ab4ad8a5098a"} {"msg":"ds_connection connected from Ok(127.0.0.1:49575)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:58.280407499Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1408,"":"io task","client":"1","":"downstairs","session_id":"39b47c96-9b2f-4901-9866-ab4ad8a5098a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:58.28093559Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1408,"client":"0","":"downstairs","session_id":"39b47c96-9b2f-4901-9866-ab4ad8a5098a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:58.28097707Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1408,"client":"1","":"downstairs","session_id":"39b47c96-9b2f-4901-9866-ab4ad8a5098a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:58.28104693Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1408,"client":"2","":"downstairs","session_id":"39b47c96-9b2f-4901-9866-ab4ad8a5098a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:58.28128366Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1408,"session_id":"39b47c96-9b2f-4901-9866-ab4ad8a5098a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:58.28132255Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1408,"session_id":"39b47c96-9b2f-4901-9866-ab4ad8a5098a"} {"msg":"[0]R flush_numbers[0..12]: [68, 93, 88, 88, 64, 93, 97, 98, 98, 98, 98, 97]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:58.28139938Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1408,"":"downstairs","":"downstairs","session_id":"39b47c96-9b2f-4901-9866-ab4ad8a5098a"} {"msg":"[0]R generation[0..12]: [49, 63, 60, 60, 47, 63, 66, 67, 67, 67, 67, 66]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:58.281410541Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1408,"":"downstairs","":"downstairs","session_id":"39b47c96-9b2f-4901-9866-ab4ad8a5098a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:58.281422111Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1408,"":"downstairs","":"downstairs","session_id":"39b47c96-9b2f-4901-9866-ab4ad8a5098a"} {"msg":"[1]R flush_numbers[0..12]: [68, 93, 88, 88, 64, 93, 97, 98, 98, 98, 98, 97]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:58.281432821Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1408,"":"downstairs","":"downstairs","session_id":"39b47c96-9b2f-4901-9866-ab4ad8a5098a"} {"msg":"[1]R generation[0..12]: [49, 63, 60, 60, 47, 63, 66, 67, 67, 67, 67, 66]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:58.281443191Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1408,"":"downstairs","":"downstairs","session_id":"39b47c96-9b2f-4901-9866-ab4ad8a5098a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:58.281454121Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1408,"":"downstairs","":"downstairs","session_id":"39b47c96-9b2f-4901-9866-ab4ad8a5098a"} {"msg":"[2]R flush_numbers[0..12]: [68, 93, 88, 88, 64, 93, 97, 98, 98, 98, 98, 97]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:58.281464661Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1408,"":"downstairs","":"downstairs","session_id":"39b47c96-9b2f-4901-9866-ab4ad8a5098a"} {"msg":"[2]R generation[0..12]: [49, 63, 60, 60, 47, 63, 66, 67, 67, 67, 67, 66]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:58.281475101Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1408,"":"downstairs","":"downstairs","session_id":"39b47c96-9b2f-4901-9866-ab4ad8a5098a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:58.281486391Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1408,"":"downstairs","":"downstairs","session_id":"39b47c96-9b2f-4901-9866-ab4ad8a5098a"} {"msg":"Max found gen is 68","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:58.281495921Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1408,"":"downstairs","":"downstairs","session_id":"39b47c96-9b2f-4901-9866-ab4ad8a5098a"} {"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:58.281505731Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1408,"":"downstairs","":"downstairs","session_id":"39b47c96-9b2f-4901-9866-ab4ad8a5098a"} {"msg":"Next flush: 99","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:58.281515091Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1408,"":"downstairs","":"downstairs","session_id":"39b47c96-9b2f-4901-9866-ab4ad8a5098a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:58.281524501Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1408,"":"downstairs","":"downstairs","session_id":"39b47c96-9b2f-4901-9866-ab4ad8a5098a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:58.281535431Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1408,"session_id":"39b47c96-9b2f-4901-9866-ab4ad8a5098a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:58.281544381Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1408,"client":"0","":"downstairs","session_id":"39b47c96-9b2f-4901-9866-ab4ad8a5098a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:58.281554501Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1408,"client":"1","":"downstairs","session_id":"39b47c96-9b2f-4901-9866-ab4ad8a5098a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:58.281564461Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1408,"client":"2","":"downstairs","session_id":"39b47c96-9b2f-4901-9866-ab4ad8a5098a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:58.281574701Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1408,"session_id":"39b47c96-9b2f-4901-9866-ab4ad8a5098a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:58.281583991Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1408,"session_id":"39b47c96-9b2f-4901-9866-ab4ad8a5098a"} {"msg":"d7cbe96d-b1d4-4cde-b1a9-23b579513138 is now active with session: 39b47c96-9b2f-4901-9866-ab4ad8a5098a","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:58.281594161Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1408,"session_id":"39b47c96-9b2f-4901-9866-ab4ad8a5098a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:58.281603011Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1408,"session_id":"39b47c96-9b2f-4901-9866-ab4ad8a5098a"} {"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:58.281612201Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1408} {"msg":"Activated sub_volume d7cbe96d-b1d4-4cde-b1a9-23b579513138","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:58.281620261Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1408} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:58.281628531Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1408,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:58.281906471Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1408,"task":"crutest"} 01/30 Write block 26 len 28672 data: 7 8 8 7 9 9 6 02/30 Write block 71 len 12288 data: 5 5 5 03/30 Read block 155 len 8192 04/30 Read block 507 len 4096 05/30 Read block 397 len 4096 06/30 Read block 192 len 20480 07/30 Write block 27 len 28672 data: 9 9 8 10 10 7 5 08/30 Write block 272 len 4096 data: 12 09/30 Write block 84 len 32768 data: 5 6 7 6 5 5 6 5 10/30 Write block 191 len 28672 data: 8 10 9 6 6 6 7 11/30 Read block 168 len 4096 12/30 Read block 480 len 36864 13/30 Read block 83 len 4096 14/30 Read block 139 len 4096 15/30 Write block 333 len 40960 data: 5 6 7 6 6 5 6 7 8 5 16/30 Write block 65 len 12288 data: 7 7 7 17/30 Read block 307 len 24576 18/30 Write block 352 len 16384 data: 5 6 7 6 19/30 Read block 20 len 40960 20/30 Read block 154 len 16384 21/30 Write block 374 len 32768 data: 9 10 11 9 9 8 7 6 22/30 Read block 481 len 28672 23/30 Write block 442 len 40960 data: 3 3 4 4 4 5 6 6 5 4 24/30 Write block 331 len 8192 data: 7 6 25/30 Read block 491 len 32768 26/30 Read block 588 len 4096 27/30 Read block 262 len 4096 28/30 Read block 566 len 8192 29/30 Read block 299 len 8192 30/30 Write block 95 len 24576 data: 7 6 7 6 5 4 ---------------------------------------------------------------- Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Write 3 Done Done Done false 1002 Acked Read 2 Done Done Done false 1003 Acked Read 1 Done Done Done false 1004 Acked Read 1 Done Done Sent false 1005 Acked Read 5 Done Done Sent false 1006 Acked Write 7 Done Done Sent false 1007 Acked Write 1 Done Done Sent false 1008 Acked Write 8 Done Done Sent false 1009 Acked Write 7 Done Done Sent false 1010 Acked Read 1 Done Done Sent false 1011 Acked Read 9 Done Done Sent false 1012 Acked Read 1 Done Done Sent false 1013 Acked Read 1 Done Done Sent false 1014 Acked Write 10 Done Done Sent false 1015 Acked Write 3 Done Done Sent false 1016 Acked Read 6 Done Done Sent false 1017 Acked Write 4 Done Done Sent false 1018 Acked Read 10 Done Done Sent false 1019 Acked Read 4 Done Done Sent false 1020 Acked Write 8 Done Done Sent false 1021 Acked Read 7 Done Done Sent false 1022 Acked Write 10 Done Done Sent false 1023 Acked Write 2 Done Done Sent false 1024 Acked Read 8 Done Done Sent false 1025 Acked Read 1 Done Done Sent false 1026 Acked Read 1 Done Done Sent false 1027 Acked Read 2 Done Done Sent false 1028 Acked Read 2 Done Done Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 26 28 Done 29 29 4 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:21:58.289671563Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1408} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:21:58.289836093Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1408,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 68 68 68 F F F 1 020-039 63 63 63 93 93 93 T T T 2 040-059 60 60 60 88 88 88 F F F 3 060-079 60 60 60 88 88 88 T T T 4 080-099 47 47 47 64 64 64 T T F <--- 5 100-119 63 63 63 93 93 93 T T F <--- 6 120-139 66 66 66 97 97 97 F F F 7 140-159 67 67 67 98 98 98 F F F 8 160-179 67 67 67 98 98 98 F F F 9 180-199 67 67 67 98 98 98 T T F <--- 10 200-219 67 67 67 98 98 98 F F F 11 220-239 66 66 66 97 97 97 F F F 12 240-259 66 66 66 97 97 97 F F F 13 260-279 66 66 66 97 97 97 T T F <--- 14 280-299 67 67 67 98 98 98 F F F 15 300-319 67 67 67 98 98 98 F F F 16 320-339 54 54 54 77 77 77 T T F <--- 17 340-359 64 64 64 95 95 95 T T F <--- 18 360-379 64 64 64 94 94 94 T T F <--- 19 380-399 67 67 67 98 98 98 T T F <--- 20 400-419 66 66 66 97 97 97 F F F 21 420-439 67 67 67 98 98 98 F F F 22 440-459 64 64 64 94 94 94 T T F <--- 23 460-479 67 67 67 98 98 98 F F F 24 480-499 64 64 64 95 95 95 F F F 25 500-519 65 65 65 96 96 96 F F F 26 520-539 65 65 65 96 96 96 F F F 27 540-559 64 64 64 94 94 94 F F F 28 560-579 64 64 64 95 95 95 F F F 29 580-599 63 63 63 93 93 93 F F F Max gen: 67, Max flush: 98 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 34 Verifying data now No change after verify Loop: 34 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 68 68 68 F F F 1 020-039 68 68 68 99 99 99 F F F 2 040-059 60 60 60 88 88 88 F F F 3 060-079 68 68 68 99 99 99 F F F 4 080-099 68 68 68 99 99 99 F F F 5 100-119 68 68 68 99 99 99 F F F 6 120-139 66 66 66 97 97 97 F F F 7 140-159 67 67 67 98 98 98 F F F 8 160-179 67 67 67 98 98 98 F F F 9 180-199 68 68 68 99 99 99 F F F 10 200-219 67 67 67 98 98 98 F F F 11 220-239 66 66 66 97 97 97 F F F 12 240-259 66 66 66 97 97 97 F F F 13 260-279 68 68 68 99 99 99 F F F 14 280-299 67 67 67 98 98 98 F F F 15 300-319 67 67 67 98 98 98 F F F 16 320-339 68 68 68 99 99 99 F F F 17 340-359 68 68 68 99 99 99 F F F 18 360-379 68 68 68 99 99 99 F F F 19 380-399 68 68 68 99 99 99 F F F 20 400-419 66 66 66 97 97 97 F F F 21 420-439 67 67 67 98 98 98 F F F 22 440-459 68 68 68 99 99 99 F F F 23 460-479 67 67 67 98 98 98 F F F 24 480-499 64 64 64 95 95 95 F F F 25 500-519 65 65 65 96 96 96 F F F 26 520-539 65 65 65 96 96 96 F F F 27 540-559 64 64 64 94 94 94 F F F 28 560-579 64 64 64 95 95 95 F F F 29 580-599 63 63 63 93 93 93 F F F Max gen: 68, Max flush: 99 Begin loop 35 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:02.539659526Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1416,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:02.547734299Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1416,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:02.548040889Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1416} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:02.548068159Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1416} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:02.548076799Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1416} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:02.548091619Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1416} {"msg":"Crucible 069cf55b-e78e-49f9-9a05-c7e8933fd3ec has session id: 0c5e821b-8a48-4ccc-8edb-79d729548975","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:02.54842041Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1416,"session_id":"0c5e821b-8a48-4ccc-8edb-79d729548975"} {"msg":"Upstairs opts: Upstairs UUID: 069cf55b-e78e-49f9-9a05-c7e8933fd3ec, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:02.54844534Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1416,"session_id":"0c5e821b-8a48-4ccc-8edb-79d729548975"} {"msg":"Crucible stats registered with UUID: 069cf55b-e78e-49f9-9a05-c7e8933fd3ec","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:02.54845567Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1416,"session_id":"0c5e821b-8a48-4ccc-8edb-79d729548975"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:22:02.54846478Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1416,"session_id":"0c5e821b-8a48-4ccc-8edb-79d729548975"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:02.54861239Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1416,"":"io task","client":"0","":"downstairs","session_id":"0c5e821b-8a48-4ccc-8edb-79d729548975"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:02.54862968Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1416,"":"io task","client":"2","":"downstairs","session_id":"0c5e821b-8a48-4ccc-8edb-79d729548975"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:02.54863986Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1416,"":"io task","client":"1","":"downstairs","session_id":"0c5e821b-8a48-4ccc-8edb-79d729548975"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:02.54878682Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1416,"task":"crutest"} {"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:02.54880347Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1416} {"msg":"069cf55b-e78e-49f9-9a05-c7e8933fd3ec active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:02.54881962Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1416,"session_id":"0c5e821b-8a48-4ccc-8edb-79d729548975"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:02.54882865Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1416,"client":"0","":"downstairs","session_id":"0c5e821b-8a48-4ccc-8edb-79d729548975"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:02.54883836Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1416,"client":"1","":"downstairs","session_id":"0c5e821b-8a48-4ccc-8edb-79d729548975"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:02.54884811Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1416,"client":"2","":"downstairs","session_id":"0c5e821b-8a48-4ccc-8edb-79d729548975"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:02.5488578Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1416,"":"io task","client":"2","":"downstairs","session_id":"0c5e821b-8a48-4ccc-8edb-79d729548975"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:02.54886978Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1416,"":"io task","client":"0","":"downstairs","session_id":"0c5e821b-8a48-4ccc-8edb-79d729548975"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:02.54888748Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1416,"":"io task","client":"1","":"downstairs","session_id":"0c5e821b-8a48-4ccc-8edb-79d729548975"} {"msg":"ds_connection connected from Ok(127.0.0.1:57219)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:02.54893583Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1416,"":"io task","client":"0","":"downstairs","session_id":"0c5e821b-8a48-4ccc-8edb-79d729548975"} {"msg":"ds_connection connected from Ok(127.0.0.1:39499)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:02.54894704Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1416,"":"io task","client":"1","":"downstairs","session_id":"0c5e821b-8a48-4ccc-8edb-79d729548975"} {"msg":"ds_connection connected from Ok(127.0.0.1:55960)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:02.54896143Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1416,"":"io task","client":"2","":"downstairs","session_id":"0c5e821b-8a48-4ccc-8edb-79d729548975"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:02.549603731Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1416,"client":"0","":"downstairs","session_id":"0c5e821b-8a48-4ccc-8edb-79d729548975"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:02.549637501Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1416,"client":"1","":"downstairs","session_id":"0c5e821b-8a48-4ccc-8edb-79d729548975"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:02.549681731Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1416,"client":"2","":"downstairs","session_id":"0c5e821b-8a48-4ccc-8edb-79d729548975"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:02.549913722Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1416,"session_id":"0c5e821b-8a48-4ccc-8edb-79d729548975"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:02.549951272Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1416,"session_id":"0c5e821b-8a48-4ccc-8edb-79d729548975"} {"msg":"[0]R flush_numbers[0..12]: [68, 99, 88, 99, 99, 99, 97, 98, 98, 99, 98, 97]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:02.549994052Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1416,"":"downstairs","":"downstairs","session_id":"0c5e821b-8a48-4ccc-8edb-79d729548975"} {"msg":"[0]R generation[0..12]: [49, 68, 60, 68, 68, 68, 66, 67, 67, 68, 67, 66]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:02.550005342Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1416,"":"downstairs","":"downstairs","session_id":"0c5e821b-8a48-4ccc-8edb-79d729548975"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:02.550016762Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1416,"":"downstairs","":"downstairs","session_id":"0c5e821b-8a48-4ccc-8edb-79d729548975"} {"msg":"[1]R flush_numbers[0..12]: [68, 99, 88, 99, 99, 99, 97, 98, 98, 99, 98, 97]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:02.550027202Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1416,"":"downstairs","":"downstairs","session_id":"0c5e821b-8a48-4ccc-8edb-79d729548975"} {"msg":"[1]R generation[0..12]: [49, 68, 60, 68, 68, 68, 66, 67, 67, 68, 67, 66]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:02.550037762Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1416,"":"downstairs","":"downstairs","session_id":"0c5e821b-8a48-4ccc-8edb-79d729548975"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:02.550051242Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1416,"":"downstairs","":"downstairs","session_id":"0c5e821b-8a48-4ccc-8edb-79d729548975"} {"msg":"[2]R flush_numbers[0..12]: [68, 99, 88, 99, 99, 99, 97, 98, 98, 99, 98, 97]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:02.550061882Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1416,"":"downstairs","":"downstairs","session_id":"0c5e821b-8a48-4ccc-8edb-79d729548975"} {"msg":"[2]R generation[0..12]: [49, 68, 60, 68, 68, 68, 66, 67, 67, 68, 67, 66]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:02.550072342Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1416,"":"downstairs","":"downstairs","session_id":"0c5e821b-8a48-4ccc-8edb-79d729548975"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:02.550083402Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1416,"":"downstairs","":"downstairs","session_id":"0c5e821b-8a48-4ccc-8edb-79d729548975"} {"msg":"Max found gen is 69","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:02.550092912Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1416,"":"downstairs","":"downstairs","session_id":"0c5e821b-8a48-4ccc-8edb-79d729548975"} {"msg":"Generation requested: 70 >= found:69","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:02.550102722Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1416,"":"downstairs","":"downstairs","session_id":"0c5e821b-8a48-4ccc-8edb-79d729548975"} {"msg":"Next flush: 100","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:02.550112012Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1416,"":"downstairs","":"downstairs","session_id":"0c5e821b-8a48-4ccc-8edb-79d729548975"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:02.550121432Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1416,"":"downstairs","":"downstairs","session_id":"0c5e821b-8a48-4ccc-8edb-79d729548975"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:02.550131222Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1416,"session_id":"0c5e821b-8a48-4ccc-8edb-79d729548975"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:02.550140172Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1416,"client":"0","":"downstairs","session_id":"0c5e821b-8a48-4ccc-8edb-79d729548975"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:02.550149962Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1416,"client":"1","":"downstairs","session_id":"0c5e821b-8a48-4ccc-8edb-79d729548975"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:02.550159802Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1416,"client":"2","":"downstairs","session_id":"0c5e821b-8a48-4ccc-8edb-79d729548975"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:02.550169702Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1416,"session_id":"0c5e821b-8a48-4ccc-8edb-79d729548975"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:02.550179032Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1416,"session_id":"0c5e821b-8a48-4ccc-8edb-79d729548975"} {"msg":"069cf55b-e78e-49f9-9a05-c7e8933fd3ec is now active with session: 0c5e821b-8a48-4ccc-8edb-79d729548975","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:02.550189122Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1416,"session_id":"0c5e821b-8a48-4ccc-8edb-79d729548975"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:02.550199202Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1416,"session_id":"0c5e821b-8a48-4ccc-8edb-79d729548975"} {"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:02.550208492Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1416} {"msg":"Activated sub_volume 069cf55b-e78e-49f9-9a05-c7e8933fd3ec","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:02.550216862Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1416} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:02.550225152Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1416,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:02.550501593Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1416,"task":"crutest"} 01/30 Write block 6 len 40960 data: 4 4 4 2 3 3 3 3 4 4 02/30 Write block 280 len 8192 data: 6 4 03/30 Read block 188 len 40960 04/30 Write block 384 len 12288 data: 5 7 7 05/30 Write block 570 len 20480 data: 7 7 9 6 5 06/30 Flush 07/30 Write block 259 len 16384 data: 6 6 5 6 08/30 Read block 167 len 12288 09/30 Flush 10/30 Write block 360 len 24576 data: 11 8 9 8 10 10 11/30 Write block 518 len 20480 data: 5 6 6 6 6 12/30 Read block 526 len 24576 13/30 Read block 482 len 24576 14/30 Read block 11 len 8192 15/30 Flush 16/30 Write block 517 len 4096 data: 4 17/30 Read block 501 len 16384 18/30 Read block 245 len 8192 19/30 Write block 239 len 4096 data: 12 20/30 Read block 336 len 28672 21/30 Read block 32 len 36864 22/30 Read block 187 len 12288 23/30 Write block 341 len 16384 data: 9 6 6 6 24/30 Flush 25/30 Write block 106 len 24576 data: 9 8 8 8 9 8 26/30 Read block 437 len 20480 27/30 Write block 493 len 32768 data: 9 7 5 5 5 5 6 5 28/30 Write block 551 len 28672 data: 3 4 4 3 5 5 5 29/30 Write block 135 len 24576 data: 4 4 5 8 11 10 30/30 Write block 168 len 8192 data: 6 6 ---------------------------------------------------------------- Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Write 2 Done Done Done false 1002 Acked Read 10 Done Done Sent false 1003 Acked Write 3 Done Done Sent false 1004 Acked Write 5 Done Done Sent false 1005 Acked Flush 0 Done Done Sent false 1006 Acked Write 4 Done Done Sent false 1007 Acked Read 3 Done Done Sent false 1008 Acked Flush 0 Done Done Sent false 1009 Acked Write 6 Done Done Sent false 1010 Acked Write 5 Done Done Sent false 1011 Acked Read 6 Done Done Sent false 1012 Acked Read 6 Done Done Sent false 1013 Acked Read 2 Done Done Sent false 1014 Acked Flush 0 Done Done Sent false 1015 Acked Write 1 Done Done Sent false 1016 Acked Read 4 Done Done Sent false 1017 Acked Read 2 Done Done Sent false 1018 Acked Write 1 Done Done Sent false 1019 Acked Read 7 Done Done Sent false 1020 Acked Read 9 Done Done Sent false 1021 Acked Read 3 Done Done Sent false 1022 Acked Write 4 Done Done Sent false 1023 Acked Flush 0 Done Done Sent false 1024 Acked Write 6 Done Done Sent false 1025 Acked Read 5 Done Done Sent false 1026 Acked Write 8 Done Done Sent false 1027 Acked Write 7 Sent Sent Sent false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 28 34 Done 27 27 2 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:22:02.558460555Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1416} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:02.558660185Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1416,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 70 70 49 100 100 68 F F T <--- 1 020-039 68 68 68 99 99 99 F F F 2 040-059 60 60 60 88 88 88 F F F 3 060-079 68 68 68 99 99 99 F F F 4 080-099 68 68 68 99 99 99 F F F 5 100-119 68 68 68 99 99 99 T T F <--- 6 120-139 66 66 66 97 97 97 T T F <--- 7 140-159 67 67 67 98 98 98 T T F <--- 8 160-179 67 67 67 98 98 98 T T F <--- 9 180-199 68 68 68 99 99 99 F F F 10 200-219 67 67 67 98 98 98 F F F 11 220-239 70 70 66 103 103 97 F F F <--- 12 240-259 70 70 66 101 101 97 F F F <--- 13 260-279 70 70 68 101 101 99 F F F <--- 14 280-299 70 70 67 100 100 98 F F T <--- 15 300-319 67 67 67 98 98 98 F F F 16 320-339 68 68 68 99 99 99 F F F 17 340-359 70 70 68 103 103 99 F F F <--- 18 360-379 70 70 68 102 102 99 F F F <--- 19 380-399 70 70 68 100 100 99 F F F <--- 20 400-419 66 66 66 97 97 97 F F F 21 420-439 67 67 67 98 98 98 F F F 22 440-459 68 68 68 99 99 99 F F F 23 460-479 67 67 67 98 98 98 F F F 24 480-499 64 64 64 95 95 95 T T F <--- 25 500-519 70 70 65 103 103 96 T T F <--- 26 520-539 70 70 65 102 102 96 F F F <--- 27 540-559 64 64 64 94 94 94 T T F <--- 28 560-579 70 70 64 100 100 95 F F F <--- 29 580-599 63 63 63 93 93 93 F F F Max gen: 70, Max flush: 103 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 35 Verifying data now No change after verify Loop: 35 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 70 70 70 100 100 100 F F F 1 020-039 68 68 68 99 99 99 F F F 2 040-059 60 60 60 88 88 88 F F F 3 060-079 68 68 68 99 99 99 F F F 4 080-099 68 68 68 99 99 99 F F F 5 100-119 71 71 71 104 104 104 F F F 6 120-139 71 71 71 104 104 104 F F F 7 140-159 71 71 71 104 104 104 F F F 8 160-179 71 71 71 104 104 104 F F F 9 180-199 68 68 68 99 99 99 F F F 10 200-219 67 67 67 98 98 98 F F F 11 220-239 70 70 70 103 103 103 F F F 12 240-259 70 70 70 101 101 101 F F F 13 260-279 70 70 70 101 101 101 F F F 14 280-299 70 70 70 100 100 100 F F F 15 300-319 67 67 67 98 98 98 F F F 16 320-339 68 68 68 99 99 99 F F F 17 340-359 70 70 70 103 103 103 F F F 18 360-379 70 70 70 102 102 102 F F F 19 380-399 70 70 70 100 100 100 F F F 20 400-419 66 66 66 97 97 97 F F F 21 420-439 67 67 67 98 98 98 F F F 22 440-459 68 68 68 99 99 99 F F F 23 460-479 67 67 67 98 98 98 F F F 24 480-499 71 71 71 104 104 104 F F F 25 500-519 71 71 71 104 104 104 F F F 26 520-539 70 70 70 102 102 102 F F F 27 540-559 71 71 71 104 104 104 F F F 28 560-579 70 70 70 100 100 100 F F F 29 580-599 63 63 63 93 93 93 F F F Max gen: 71, Max flush: 104 Begin loop 36 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:06.814505037Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1424,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:06.822608869Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1424,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:06.822940169Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1424} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:06.822968219Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1424} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:06.82297689Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1424} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:06.82298886Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1424} {"msg":"Crucible 434b6d88-7851-4fa5-b5de-2f470a6b1810 has session id: 1f2fdf23-fc48-409f-8c21-541c065136d4","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:06.82329758Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1424,"session_id":"1f2fdf23-fc48-409f-8c21-541c065136d4"} {"msg":"Upstairs opts: Upstairs UUID: 434b6d88-7851-4fa5-b5de-2f470a6b1810, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:06.82335869Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1424,"session_id":"1f2fdf23-fc48-409f-8c21-541c065136d4"} {"msg":"Crucible stats registered with UUID: 434b6d88-7851-4fa5-b5de-2f470a6b1810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:06.82338566Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1424,"session_id":"1f2fdf23-fc48-409f-8c21-541c065136d4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:22:06.82339843Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1424,"session_id":"1f2fdf23-fc48-409f-8c21-541c065136d4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:06.82353818Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1424,"":"io task","client":"2","":"downstairs","session_id":"1f2fdf23-fc48-409f-8c21-541c065136d4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:06.82355619Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1424,"":"io task","client":"0","":"downstairs","session_id":"1f2fdf23-fc48-409f-8c21-541c065136d4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:06.8235664Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1424,"":"io task","client":"1","":"downstairs","session_id":"1f2fdf23-fc48-409f-8c21-541c065136d4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:06.823704411Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1424,"task":"crutest"} {"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:06.823719721Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1424} {"msg":"434b6d88-7851-4fa5-b5de-2f470a6b1810 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:06.823754211Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1424,"session_id":"1f2fdf23-fc48-409f-8c21-541c065136d4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:06.823763711Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1424,"client":"0","":"downstairs","session_id":"1f2fdf23-fc48-409f-8c21-541c065136d4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:06.823773461Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1424,"client":"1","":"downstairs","session_id":"1f2fdf23-fc48-409f-8c21-541c065136d4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:06.823783191Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1424,"client":"2","":"downstairs","session_id":"1f2fdf23-fc48-409f-8c21-541c065136d4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:06.823792931Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1424,"":"io task","client":"2","":"downstairs","session_id":"1f2fdf23-fc48-409f-8c21-541c065136d4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:06.823803011Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1424,"":"io task","client":"0","":"downstairs","session_id":"1f2fdf23-fc48-409f-8c21-541c065136d4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:06.823813111Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1424,"":"io task","client":"1","":"downstairs","session_id":"1f2fdf23-fc48-409f-8c21-541c065136d4"} {"msg":"ds_connection connected from Ok(127.0.0.1:55587)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:06.823878831Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1424,"":"io task","client":"2","":"downstairs","session_id":"1f2fdf23-fc48-409f-8c21-541c065136d4"} {"msg":"ds_connection connected from Ok(127.0.0.1:35615)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:06.823899081Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1424,"":"io task","client":"0","":"downstairs","session_id":"1f2fdf23-fc48-409f-8c21-541c065136d4"} {"msg":"ds_connection connected from Ok(127.0.0.1:45324)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:06.823909861Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1424,"":"io task","client":"1","":"downstairs","session_id":"1f2fdf23-fc48-409f-8c21-541c065136d4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:06.824540992Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1424,"client":"0","":"downstairs","session_id":"1f2fdf23-fc48-409f-8c21-541c065136d4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:06.824573282Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1424,"client":"1","":"downstairs","session_id":"1f2fdf23-fc48-409f-8c21-541c065136d4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:06.824596562Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1424,"client":"2","":"downstairs","session_id":"1f2fdf23-fc48-409f-8c21-541c065136d4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:06.824842912Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1424,"session_id":"1f2fdf23-fc48-409f-8c21-541c065136d4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:06.824873782Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1424,"session_id":"1f2fdf23-fc48-409f-8c21-541c065136d4"} {"msg":"[0]R flush_numbers[0..12]: [100, 99, 88, 99, 99, 104, 104, 104, 104, 99, 98, 103]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:06.824928912Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1424,"":"downstairs","":"downstairs","session_id":"1f2fdf23-fc48-409f-8c21-541c065136d4"} {"msg":"[0]R generation[0..12]: [70, 68, 60, 68, 68, 71, 71, 71, 71, 68, 67, 70]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:06.824940492Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1424,"":"downstairs","":"downstairs","session_id":"1f2fdf23-fc48-409f-8c21-541c065136d4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:06.824951712Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1424,"":"downstairs","":"downstairs","session_id":"1f2fdf23-fc48-409f-8c21-541c065136d4"} {"msg":"[1]R flush_numbers[0..12]: [100, 99, 88, 99, 99, 104, 104, 104, 104, 99, 98, 103]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:06.824962273Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1424,"":"downstairs","":"downstairs","session_id":"1f2fdf23-fc48-409f-8c21-541c065136d4"} {"msg":"[1]R generation[0..12]: [70, 68, 60, 68, 68, 71, 71, 71, 71, 68, 67, 70]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:06.824972773Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1424,"":"downstairs","":"downstairs","session_id":"1f2fdf23-fc48-409f-8c21-541c065136d4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:06.824983893Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1424,"":"downstairs","":"downstairs","session_id":"1f2fdf23-fc48-409f-8c21-541c065136d4"} {"msg":"[2]R flush_numbers[0..12]: [100, 99, 88, 99, 99, 104, 104, 104, 104, 99, 98, 103]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:06.824994473Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1424,"":"downstairs","":"downstairs","session_id":"1f2fdf23-fc48-409f-8c21-541c065136d4"} {"msg":"[2]R generation[0..12]: [70, 68, 60, 68, 68, 71, 71, 71, 71, 68, 67, 70]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:06.825005363Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1424,"":"downstairs","":"downstairs","session_id":"1f2fdf23-fc48-409f-8c21-541c065136d4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:06.825016433Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1424,"":"downstairs","":"downstairs","session_id":"1f2fdf23-fc48-409f-8c21-541c065136d4"} {"msg":"Max found gen is 72","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:06.825027963Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1424,"":"downstairs","":"downstairs","session_id":"1f2fdf23-fc48-409f-8c21-541c065136d4"} {"msg":"Generation requested: 72 >= found:72","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:06.825037713Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1424,"":"downstairs","":"downstairs","session_id":"1f2fdf23-fc48-409f-8c21-541c065136d4"} {"msg":"Next flush: 105","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:06.825047223Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1424,"":"downstairs","":"downstairs","session_id":"1f2fdf23-fc48-409f-8c21-541c065136d4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:06.825056673Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1424,"":"downstairs","":"downstairs","session_id":"1f2fdf23-fc48-409f-8c21-541c065136d4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:06.825066563Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1424,"session_id":"1f2fdf23-fc48-409f-8c21-541c065136d4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:06.825075473Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1424,"client":"0","":"downstairs","session_id":"1f2fdf23-fc48-409f-8c21-541c065136d4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:06.825085123Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1424,"client":"1","":"downstairs","session_id":"1f2fdf23-fc48-409f-8c21-541c065136d4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:06.825094853Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1424,"client":"2","":"downstairs","session_id":"1f2fdf23-fc48-409f-8c21-541c065136d4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:06.825104773Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1424,"session_id":"1f2fdf23-fc48-409f-8c21-541c065136d4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:06.825113953Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1424,"session_id":"1f2fdf23-fc48-409f-8c21-541c065136d4"} {"msg":"434b6d88-7851-4fa5-b5de-2f470a6b1810 is now active with session: 1f2fdf23-fc48-409f-8c21-541c065136d4","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:06.825124113Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1424,"session_id":"1f2fdf23-fc48-409f-8c21-541c065136d4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:06.825132933Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1424,"session_id":"1f2fdf23-fc48-409f-8c21-541c065136d4"} {"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:06.825142123Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1424} {"msg":"Activated sub_volume 434b6d88-7851-4fa5-b5de-2f470a6b1810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:06.825150303Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1424} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:06.825158563Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1424,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:06.825460133Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1424,"task":"crutest"} 01/30 Flush 02/30 Write block 467 len 8192 data: 6 9 03/30 Write block 148 len 4096 data: 7 04/30 Read block 31 len 20480 05/30 Write block 420 len 12288 data: 5 7 7 06/30 Write block 161 len 16384 data: 7 4 3 4 07/30 Write block 128 len 4096 data: 4 08/30 Read block 285 len 24576 09/30 Read block 363 len 40960 10/30 Read block 582 len 8192 11/30 Write block 290 len 28672 data: 6 4 5 8 7 8 8 12/30 Write block 57 len 24576 data: 6 7 6 7 8 7 13/30 Read block 579 len 16384 14/30 Write block 472 len 40960 data: 4 4 5 5 6 6 6 6 6 5 15/30 Write block 356 len 12288 data: 5 6 7 16/30 Read block 190 len 32768 17/30 Write block 249 len 20480 data: 6 5 5 4 4 18/30 Write block 551 len 28672 data: 4 5 5 4 6 6 6 19/30 Flush 20/30 Write block 114 len 20480 data: 6 5 5 3 5 21/30 Read block 231 len 32768 22/30 Write block 588 len 32768 data: 8 6 4 3 4 4 4 5 23/30 Flush 24/30 Write block 12 len 16384 data: 4 4 5 5 25/30 Read block 584 len 12288 26/30 Read block 414 len 8192 27/30 Read block 239 len 24576 28/30 Read block 356 len 16384 29/30 Write block 3 len 8192 data: 4 4 30/30 Write block 336 len 20480 data: 7 7 6 7 8 ---------------------------------------------------------------- Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Flush 0 Done Sent Done false 1001 Acked Write 2 Done Sent Done false 1002 Acked Write 1 Done Sent Done false 1003 Acked Read 5 Done Sent Done false 1004 Acked Write 3 Done Sent Done false 1005 Acked Write 4 Done Sent Done false 1006 Acked Write 1 Done Sent Done false 1007 Acked Read 6 Done Sent Done false 1008 Acked Read 10 Done Sent Done false 1009 Acked Read 2 Done Sent Done false 1010 Acked Write 7 Done Sent Done false 1011 Acked Write 6 Done Sent Done false 1012 Acked Read 4 Done Sent Done false 1013 Acked Write 10 Done Sent Done false 1014 Acked Write 3 Done Sent Done false 1015 Acked Read 8 Done Sent Done false 1016 Acked Write 5 Done Sent Done false 1017 Acked Write 7 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Write 5 Done Sent Done false 1020 Acked Read 8 Done Sent Done false 1021 Acked Write 8 Done Sent Done false 1022 Acked Flush 0 Done Sent Done false 1023 Acked Write 4 Done Sent Done false 1024 Acked Read 3 Done Sent Done false 1025 Acked Read 2 Done Sent Done false 1026 Acked Read 6 Done Sent Done false 1027 Acked Read 4 Done Sent Done false 1028 Acked Write 2 Done Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 30 2 33 Done 29 0 28 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) None Some(JobId(1022)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:22:06.833812436Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1424} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:06.833995486Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1424,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 70 70 70 100 100 100 T F T <--- 1 020-039 68 68 68 99 99 99 F F F 2 040-059 72 60 72 106 88 106 F F F <--- 3 060-079 72 68 72 106 99 106 F F F <--- 4 080-099 68 68 68 99 99 99 F F F 5 100-119 72 71 72 107 104 107 F F F <--- 6 120-139 72 71 72 106 104 106 F F F <--- 7 140-159 72 71 72 106 104 106 F F F <--- 8 160-179 72 71 72 106 104 106 F F F <--- 9 180-199 68 68 68 99 99 99 F F F 10 200-219 67 67 67 98 98 98 F F F 11 220-239 70 70 70 103 103 103 F F F 12 240-259 72 70 72 106 101 106 F F F <--- 13 260-279 70 70 70 101 101 101 F F F 14 280-299 72 70 72 106 100 106 F F F <--- 15 300-319 67 67 67 98 98 98 F F F 16 320-339 68 68 68 99 99 99 T F T <--- 17 340-359 72 70 72 106 103 106 T F T <--- 18 360-379 70 70 70 102 102 102 F F F 19 380-399 70 70 70 100 100 100 F F F 20 400-419 66 66 66 97 97 97 F F F 21 420-439 72 67 72 106 98 106 F F F <--- 22 440-459 68 68 68 99 99 99 F F F 23 460-479 72 67 72 106 98 106 F F F <--- 24 480-499 72 71 72 106 104 106 F F F <--- 25 500-519 71 71 71 104 104 104 F F F 26 520-539 70 70 70 102 102 102 F F F 27 540-559 72 71 72 106 104 106 F F F <--- 28 560-579 70 70 70 100 100 100 F F F 29 580-599 72 63 72 107 93 107 F F F <--- Max gen: 72, Max flush: 107 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 36 Verifying data now No change after verify Loop: 36 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 73 73 73 108 108 108 F F F 1 020-039 68 68 68 99 99 99 F F F 2 040-059 72 72 72 106 106 106 F F F 3 060-079 72 72 72 106 106 106 F F F 4 080-099 68 68 68 99 99 99 F F F 5 100-119 72 72 72 107 107 107 F F F 6 120-139 72 72 72 106 106 106 F F F 7 140-159 72 72 72 106 106 106 F F F 8 160-179 72 72 72 106 106 106 F F F 9 180-199 68 68 68 99 99 99 F F F 10 200-219 67 67 67 98 98 98 F F F 11 220-239 70 70 70 103 103 103 F F F 12 240-259 72 72 72 106 106 106 F F F 13 260-279 70 70 70 101 101 101 F F F 14 280-299 72 72 72 106 106 106 F F F 15 300-319 67 67 67 98 98 98 F F F 16 320-339 73 73 73 108 108 108 F F F 17 340-359 73 73 73 108 108 108 F F F 18 360-379 70 70 70 102 102 102 F F F 19 380-399 70 70 70 100 100 100 F F F 20 400-419 66 66 66 97 97 97 F F F 21 420-439 72 72 72 106 106 106 F F F 22 440-459 68 68 68 99 99 99 F F F 23 460-479 72 72 72 106 106 106 F F F 24 480-499 72 72 72 106 106 106 F F F 25 500-519 71 71 71 104 104 104 F F F 26 520-539 70 70 70 102 102 102 F F F 27 540-559 72 72 72 106 106 106 F F F 28 560-579 70 70 70 100 100 100 F F F 29 580-599 72 72 72 107 107 107 F F F Max gen: 73, Max flush: 108 Begin loop 37 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:11.088578816Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1432,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:11.096590059Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1432,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:11.096876029Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1432} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:11.096903779Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1432} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:11.096912489Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1432} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:11.096924129Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1432} {"msg":"Crucible ac197151-8f49-4cf2-bca1-317df2f38cb9 has session id: dde50af9-9059-4693-862b-fcdd1746e668","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:11.09726335Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1432,"session_id":"dde50af9-9059-4693-862b-fcdd1746e668"} {"msg":"Upstairs opts: Upstairs UUID: ac197151-8f49-4cf2-bca1-317df2f38cb9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:11.09729329Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1432,"session_id":"dde50af9-9059-4693-862b-fcdd1746e668"} {"msg":"Crucible stats registered with UUID: ac197151-8f49-4cf2-bca1-317df2f38cb9","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:11.09730448Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1432,"session_id":"dde50af9-9059-4693-862b-fcdd1746e668"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:22:11.09731383Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1432,"session_id":"dde50af9-9059-4693-862b-fcdd1746e668"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:11.09742546Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1432,"":"io task","client":"1","":"downstairs","session_id":"dde50af9-9059-4693-862b-fcdd1746e668"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:11.09744656Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1432,"":"io task","client":"0","":"downstairs","session_id":"dde50af9-9059-4693-862b-fcdd1746e668"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:11.09745686Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1432,"":"io task","client":"2","":"downstairs","session_id":"dde50af9-9059-4693-862b-fcdd1746e668"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:11.09761194Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1432,"task":"crutest"} {"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:11.09763078Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1432} {"msg":"ac197151-8f49-4cf2-bca1-317df2f38cb9 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:11.09766121Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1432,"session_id":"dde50af9-9059-4693-862b-fcdd1746e668"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:11.09767254Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1432,"client":"0","":"downstairs","session_id":"dde50af9-9059-4693-862b-fcdd1746e668"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:11.09768224Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1432,"client":"1","":"downstairs","session_id":"dde50af9-9059-4693-862b-fcdd1746e668"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:11.09769207Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1432,"client":"2","":"downstairs","session_id":"dde50af9-9059-4693-862b-fcdd1746e668"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:11.09770164Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1432,"":"io task","client":"2","":"downstairs","session_id":"dde50af9-9059-4693-862b-fcdd1746e668"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:11.09771188Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1432,"":"io task","client":"0","":"downstairs","session_id":"dde50af9-9059-4693-862b-fcdd1746e668"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:11.09772185Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1432,"":"io task","client":"1","":"downstairs","session_id":"dde50af9-9059-4693-862b-fcdd1746e668"} {"msg":"ds_connection connected from Ok(127.0.0.1:39128)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:11.09776204Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1432,"":"io task","client":"1","":"downstairs","session_id":"dde50af9-9059-4693-862b-fcdd1746e668"} {"msg":"ds_connection connected from Ok(127.0.0.1:44243)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:11.09779286Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1432,"":"io task","client":"2","":"downstairs","session_id":"dde50af9-9059-4693-862b-fcdd1746e668"} {"msg":"ds_connection connected from Ok(127.0.0.1:45209)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:11.0978149Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1432,"":"io task","client":"0","":"downstairs","session_id":"dde50af9-9059-4693-862b-fcdd1746e668"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:11.098425101Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1432,"client":"0","":"downstairs","session_id":"dde50af9-9059-4693-862b-fcdd1746e668"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:11.098439721Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1432,"client":"1","":"downstairs","session_id":"dde50af9-9059-4693-862b-fcdd1746e668"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:11.098475081Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1432,"client":"2","":"downstairs","session_id":"dde50af9-9059-4693-862b-fcdd1746e668"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:11.098745122Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1432,"session_id":"dde50af9-9059-4693-862b-fcdd1746e668"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:11.098754732Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1432,"session_id":"dde50af9-9059-4693-862b-fcdd1746e668"} {"msg":"[0]R flush_numbers[0..12]: [108, 99, 106, 106, 99, 107, 106, 106, 106, 99, 98, 103]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:11.098779872Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1432,"":"downstairs","":"downstairs","session_id":"dde50af9-9059-4693-862b-fcdd1746e668"} {"msg":"[0]R generation[0..12]: [73, 68, 72, 72, 68, 72, 72, 72, 72, 68, 67, 70]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:11.098793372Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1432,"":"downstairs","":"downstairs","session_id":"dde50af9-9059-4693-862b-fcdd1746e668"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:11.098804622Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1432,"":"downstairs","":"downstairs","session_id":"dde50af9-9059-4693-862b-fcdd1746e668"} {"msg":"[1]R flush_numbers[0..12]: [108, 99, 106, 106, 99, 107, 106, 106, 106, 99, 98, 103]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:11.098815492Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1432,"":"downstairs","":"downstairs","session_id":"dde50af9-9059-4693-862b-fcdd1746e668"} {"msg":"[1]R generation[0..12]: [73, 68, 72, 72, 68, 72, 72, 72, 72, 68, 67, 70]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:11.098825972Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1432,"":"downstairs","":"downstairs","session_id":"dde50af9-9059-4693-862b-fcdd1746e668"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:11.098836972Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1432,"":"downstairs","":"downstairs","session_id":"dde50af9-9059-4693-862b-fcdd1746e668"} {"msg":"[2]R flush_numbers[0..12]: [108, 99, 106, 106, 99, 107, 106, 106, 106, 99, 98, 103]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:11.098847692Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1432,"":"downstairs","":"downstairs","session_id":"dde50af9-9059-4693-862b-fcdd1746e668"} {"msg":"[2]R generation[0..12]: [73, 68, 72, 72, 68, 72, 72, 72, 72, 68, 67, 70]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:11.098858062Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1432,"":"downstairs","":"downstairs","session_id":"dde50af9-9059-4693-862b-fcdd1746e668"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:11.098869022Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1432,"":"downstairs","":"downstairs","session_id":"dde50af9-9059-4693-862b-fcdd1746e668"} {"msg":"Max found gen is 74","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:11.098878642Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1432,"":"downstairs","":"downstairs","session_id":"dde50af9-9059-4693-862b-fcdd1746e668"} {"msg":"Generation requested: 74 >= found:74","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:11.098888442Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1432,"":"downstairs","":"downstairs","session_id":"dde50af9-9059-4693-862b-fcdd1746e668"} {"msg":"Next flush: 109","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:11.098897922Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1432,"":"downstairs","":"downstairs","session_id":"dde50af9-9059-4693-862b-fcdd1746e668"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:11.098907312Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1432,"":"downstairs","":"downstairs","session_id":"dde50af9-9059-4693-862b-fcdd1746e668"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:11.098917132Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1432,"session_id":"dde50af9-9059-4693-862b-fcdd1746e668"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:11.098926032Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1432,"client":"0","":"downstairs","session_id":"dde50af9-9059-4693-862b-fcdd1746e668"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:11.098937052Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1432,"client":"1","":"downstairs","session_id":"dde50af9-9059-4693-862b-fcdd1746e668"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:11.098946592Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1432,"client":"2","":"downstairs","session_id":"dde50af9-9059-4693-862b-fcdd1746e668"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:11.098956502Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1432,"session_id":"dde50af9-9059-4693-862b-fcdd1746e668"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:11.098965782Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1432,"session_id":"dde50af9-9059-4693-862b-fcdd1746e668"} {"msg":"ac197151-8f49-4cf2-bca1-317df2f38cb9 is now active with session: dde50af9-9059-4693-862b-fcdd1746e668","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:11.098975822Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1432,"session_id":"dde50af9-9059-4693-862b-fcdd1746e668"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:11.098984762Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1432,"session_id":"dde50af9-9059-4693-862b-fcdd1746e668"} {"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:11.098993972Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1432} {"msg":"Activated sub_volume ac197151-8f49-4cf2-bca1-317df2f38cb9","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:11.099001982Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1432} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:11.099010172Z","hostname":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 w-01KY5YGK9V792E6WY6KV93YHRV","pid":1432,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:11.099269623Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1432,"task":"crutest"} 01/30 Write block 85 len 20480 data: 7 8 7 6 6 02/30 Write block 460 len 8192 data: 4 4 03/30 Write block 154 len 4096 data: 8 04/30 Read block 224 len 32768 05/30 Read block 21 len 12288 06/30 Read block 447 len 24576 07/30 Flush 08/30 Write block 484 len 16384 data: 7 8 8 9 09/30 Write block 61 len 40960 data: 9 8 7 7 8 8 8 6 6 6 10/30 Write block 521 len 32768 data: 7 7 6 6 10 9 11 12 11/30 Read block 156 len 24576 12/30 Read block 195 len 8192 13/30 Write block 340 len 32768 data: 9 10 7 7 7 7 8 9 14/30 Read block 484 len 12288 15/30 Write block 251 len 16384 data: 6 5 5 5 16/30 Write block 300 len 4096 data: 8 17/30 Read block 302 len 20480 18/30 Write block 540 len 24576 data: 6 6 7 7 7 5 19/30 Read block 126 len 16384 20/30 Read block 590 len 8192 21/30 Read block 17 len 36864 22/30 Write block 113 len 20480 data: 7 7 6 6 4 23/30 Flush 24/30 Write block 318 len 28672 data: 6 6 8 7 9 9 8 25/30 Write block 379 len 12288 data: 9 8 7 26/30 Write block 86 len 16384 data: 9 8 7 7 27/30 Read block 24 len 12288 28/30 Read block 150 len 16384 29/30 Write block 261 len 20480 data: 6 7 9 8 9 30/30 Write block 151 len 4096 data: 8 ---------------------------------------------------------------- Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Write 2 Done Done Done false 1002 Acked Write 1 Done Done Done false 1003 Acked Read 8 Done Done Done false 1004 Acked Read 3 Done Done Sent false 1005 Acked Read 6 Done Done Sent false 1006 Acked Flush 0 Done Done Sent false 1007 Acked Write 4 Done Done Sent false 1008 Acked Write 10 Done Done Sent false 1009 Acked Write 8 Done Done Sent false 1010 Acked Read 6 Done Done Sent false 1011 Acked Read 2 Done Done Sent false 1012 Acked Write 8 Done Done Sent false 1013 Acked Read 3 Done Done Sent false 1014 Acked Write 4 Done Done Sent false 1015 Acked Write 1 Done Done Sent false 1016 Acked Read 5 Done Done Sent false 1017 Acked Write 6 Done Done Sent false 1018 Acked Read 4 Done Done Sent false 1019 Acked Read 2 Done Done Sent false 1020 Acked Read 9 Done Done Sent false 1021 Acked Write 5 Done Done Sent false 1022 Acked Flush 0 Done Done Sent false 1023 Acked Write 7 Done Done Sent false 1024 Acked Write 3 Done Done Sent false 1025 Acked Write 4 Done Done Sent false 1026 Acked Read 3 Done Done Sent false 1027 Acked Read 4 Done Done Sent false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 26 30 Done 28 28 4 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:22:11.107165044Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1432} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:11.107379385Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1432,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 73 73 73 108 108 108 F F F 1 020-039 68 68 68 99 99 99 F F F 2 040-059 72 72 72 106 106 106 F F F 3 060-079 74 74 72 110 110 106 F F F <--- 4 080-099 74 74 68 109 109 99 T T T <--- 5 100-119 74 74 72 110 110 107 F F F <--- 6 120-139 72 72 72 106 106 106 F F F 7 140-159 74 74 72 109 109 106 T T T <--- 8 160-179 72 72 72 106 106 106 F F F 9 180-199 68 68 68 99 99 99 F F F 10 200-219 67 67 67 98 98 98 F F F 11 220-239 70 70 70 103 103 103 F F F 12 240-259 74 74 72 110 110 106 F F F <--- 13 260-279 70 70 70 101 101 101 T T F <--- 14 280-299 72 72 72 106 106 106 F F F 15 300-319 74 74 67 110 110 98 T T F <--- 16 320-339 73 73 73 108 108 108 T T F <--- 17 340-359 74 74 73 110 110 108 F F F <--- 18 360-379 70 70 70 102 102 102 T T F <--- 19 380-399 70 70 70 100 100 100 T T F <--- 20 400-419 66 66 66 97 97 97 F F F 21 420-439 72 72 72 106 106 106 F F F 22 440-459 68 68 68 99 99 99 F F F 23 460-479 74 74 72 109 109 106 F F T <--- 24 480-499 74 74 72 110 110 106 F F F <--- 25 500-519 71 71 71 104 104 104 F F F 26 520-539 74 74 70 110 110 102 F F F <--- 27 540-559 74 74 72 110 110 106 F F F <--- 28 560-579 70 70 70 100 100 100 F F F 29 580-599 72 72 72 107 107 107 F F F Max gen: 74, Max flush: 110 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 37 Verifying data now No change after verify Loop: 37 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 73 73 73 108 108 108 F F F 1 020-039 68 68 68 99 99 99 F F F 2 040-059 72 72 72 106 106 106 F F F 3 060-079 74 74 74 110 110 110 F F F 4 080-099 75 75 75 111 111 111 F F F 5 100-119 74 74 74 110 110 110 F F F 6 120-139 72 72 72 106 106 106 F F F 7 140-159 75 75 75 111 111 111 F F F 8 160-179 72 72 72 106 106 106 F F F 9 180-199 68 68 68 99 99 99 F F F 10 200-219 67 67 67 98 98 98 F F F 11 220-239 70 70 70 103 103 103 F F F 12 240-259 74 74 74 110 110 110 F F F 13 260-279 75 75 75 111 111 111 F F F 14 280-299 72 72 72 106 106 106 F F F 15 300-319 75 75 75 111 111 111 F F F 16 320-339 75 75 75 111 111 111 F F F 17 340-359 74 74 74 110 110 110 F F F 18 360-379 75 75 75 111 111 111 F F F 19 380-399 75 75 75 111 111 111 F F F 20 400-419 66 66 66 97 97 97 F F F 21 420-439 72 72 72 106 106 106 F F F 22 440-459 68 68 68 99 99 99 F F F 23 460-479 74 74 74 109 109 109 F F F 24 480-499 74 74 74 110 110 110 F F F 25 500-519 71 71 71 104 104 104 F F F 26 520-539 74 74 74 110 110 110 F F F 27 540-559 74 74 74 110 110 110 F F F 28 560-579 70 70 70 100 100 100 F F F 29 580-599 72 72 72 107 107 107 F F F Max gen: 75, Max flush: 111 Begin loop 38 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:15.36493159Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1440,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:22:15.37205822Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1440,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:15.37211333Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1440,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:15.379029011Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1440,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:15.379298091Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1440} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:15.379326281Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1440} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:15.379337051Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1440} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:15.379349001Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1440} {"msg":"Crucible 25c0fd56-23b5-490d-9af6-d806e4eb17d3 has session id: f214d456-4a99-4314-a02f-a97d94ccdb91","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:15.379679872Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1440,"session_id":"f214d456-4a99-4314-a02f-a97d94ccdb91"} {"msg":"Upstairs opts: Upstairs UUID: 25c0fd56-23b5-490d-9af6-d806e4eb17d3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:15.379739942Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1440,"session_id":"f214d456-4a99-4314-a02f-a97d94ccdb91"} {"msg":"Crucible stats registered with UUID: 25c0fd56-23b5-490d-9af6-d806e4eb17d3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:15.379750802Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1440,"session_id":"f214d456-4a99-4314-a02f-a97d94ccdb91"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:22:15.379759952Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1440,"session_id":"f214d456-4a99-4314-a02f-a97d94ccdb91"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:15.379889062Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1440,"":"io task","client":"1","":"downstairs","session_id":"f214d456-4a99-4314-a02f-a97d94ccdb91"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:15.379906862Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1440,"":"io task","client":"0","":"downstairs","session_id":"f214d456-4a99-4314-a02f-a97d94ccdb91"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:15.379917122Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1440,"":"io task","client":"2","":"downstairs","session_id":"f214d456-4a99-4314-a02f-a97d94ccdb91"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:15.380093282Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1440,"task":"crutest"} {"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:15.380110022Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1440} {"msg":"25c0fd56-23b5-490d-9af6-d806e4eb17d3 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:15.380127913Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1440,"session_id":"f214d456-4a99-4314-a02f-a97d94ccdb91"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:15.380136973Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1440,"client":"0","":"downstairs","session_id":"f214d456-4a99-4314-a02f-a97d94ccdb91"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:15.380146813Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1440,"client":"1","":"downstairs","session_id":"f214d456-4a99-4314-a02f-a97d94ccdb91"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:15.380156533Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1440,"client":"2","":"downstairs","session_id":"f214d456-4a99-4314-a02f-a97d94ccdb91"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:15.380167903Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1440,"":"io task","client":"2","":"downstairs","session_id":"f214d456-4a99-4314-a02f-a97d94ccdb91"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:15.380196193Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1440,"":"io task","client":"0","":"downstairs","session_id":"f214d456-4a99-4314-a02f-a97d94ccdb91"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:15.380218483Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1440,"":"io task","client":"1","":"downstairs","session_id":"f214d456-4a99-4314-a02f-a97d94ccdb91"} {"msg":"ds_connection connected from Ok(127.0.0.1:48814)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:15.380269403Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1440,"":"io task","client":"2","":"downstairs","session_id":"f214d456-4a99-4314-a02f-a97d94ccdb91"} {"msg":"ds_connection connected from Ok(127.0.0.1:50363)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:15.380342463Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1440,"":"io task","client":"1","":"downstairs","session_id":"f214d456-4a99-4314-a02f-a97d94ccdb91"} {"msg":"ds_connection connected from Ok(127.0.0.1:59177)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:15.380383973Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1440,"":"io task","client":"0","":"downstairs","session_id":"f214d456-4a99-4314-a02f-a97d94ccdb91"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:15.380896304Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1440,"client":"1","":"downstairs","session_id":"f214d456-4a99-4314-a02f-a97d94ccdb91"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:15.380910664Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1440,"client":"2","":"downstairs","session_id":"f214d456-4a99-4314-a02f-a97d94ccdb91"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:15.380987344Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1440,"client":"0","":"downstairs","session_id":"f214d456-4a99-4314-a02f-a97d94ccdb91"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:15.381210514Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1440,"session_id":"f214d456-4a99-4314-a02f-a97d94ccdb91"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:15.381220184Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1440,"session_id":"f214d456-4a99-4314-a02f-a97d94ccdb91"} {"msg":"[0]R flush_numbers[0..12]: [108, 99, 106, 110, 111, 110, 106, 111, 106, 99, 98, 103]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:15.381348084Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1440,"":"downstairs","":"downstairs","session_id":"f214d456-4a99-4314-a02f-a97d94ccdb91"} {"msg":"[0]R generation[0..12]: [73, 68, 72, 74, 75, 74, 72, 75, 72, 68, 67, 70]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:15.381359314Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1440,"":"downstairs","":"downstairs","session_id":"f214d456-4a99-4314-a02f-a97d94ccdb91"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:15.381370684Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1440,"":"downstairs","":"downstairs","session_id":"f214d456-4a99-4314-a02f-a97d94ccdb91"} {"msg":"[1]R flush_numbers[0..12]: [108, 99, 106, 110, 111, 110, 106, 111, 106, 99, 98, 103]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:15.381381484Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1440,"":"downstairs","":"downstairs","session_id":"f214d456-4a99-4314-a02f-a97d94ccdb91"} {"msg":"[1]R generation[0..12]: [73, 68, 72, 74, 75, 74, 72, 75, 72, 68, 67, 70]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:15.381394214Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1440,"":"downstairs","":"downstairs","session_id":"f214d456-4a99-4314-a02f-a97d94ccdb91"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:15.381405334Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1440,"":"downstairs","":"downstairs","session_id":"f214d456-4a99-4314-a02f-a97d94ccdb91"} {"msg":"[2]R flush_numbers[0..12]: [108, 99, 106, 110, 111, 110, 106, 111, 106, 99, 98, 103]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:15.381416074Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1440,"":"downstairs","":"downstairs","session_id":"f214d456-4a99-4314-a02f-a97d94ccdb91"} {"msg":"[2]R generation[0..12]: [73, 68, 72, 74, 75, 74, 72, 75, 72, 68, 67, 70]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:15.381426554Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1440,"":"downstairs","":"downstairs","session_id":"f214d456-4a99-4314-a02f-a97d94ccdb91"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:15.381437604Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1440,"":"downstairs","":"downstairs","session_id":"f214d456-4a99-4314-a02f-a97d94ccdb91"} {"msg":"Max found gen is 76","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:15.381447135Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1440,"":"downstairs","":"downstairs","session_id":"f214d456-4a99-4314-a02f-a97d94ccdb91"} {"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:15.381457035Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1440,"":"downstairs","":"downstairs","session_id":"f214d456-4a99-4314-a02f-a97d94ccdb91"} {"msg":"Next flush: 112","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:15.381466525Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1440,"":"downstairs","":"downstairs","session_id":"f214d456-4a99-4314-a02f-a97d94ccdb91"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:15.381476175Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1440,"":"downstairs","":"downstairs","session_id":"f214d456-4a99-4314-a02f-a97d94ccdb91"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:15.381486095Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1440,"session_id":"f214d456-4a99-4314-a02f-a97d94ccdb91"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:15.381494925Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1440,"client":"0","":"downstairs","session_id":"f214d456-4a99-4314-a02f-a97d94ccdb91"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:15.381504695Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1440,"client":"1","":"downstairs","session_id":"f214d456-4a99-4314-a02f-a97d94ccdb91"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:15.381514415Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1440,"client":"2","":"downstairs","session_id":"f214d456-4a99-4314-a02f-a97d94ccdb91"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:15.381524575Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1440,"session_id":"f214d456-4a99-4314-a02f-a97d94ccdb91"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:15.381533885Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1440,"session_id":"f214d456-4a99-4314-a02f-a97d94ccdb91"} {"msg":"25c0fd56-23b5-490d-9af6-d806e4eb17d3 is now active with session: f214d456-4a99-4314-a02f-a97d94ccdb91","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:15.381545395Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1440,"session_id":"f214d456-4a99-4314-a02f-a97d94ccdb91"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:15.381554195Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1440,"session_id":"f214d456-4a99-4314-a02f-a97d94ccdb91"} {"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:15.381563565Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1440} {"msg":"Activated sub_volume 25c0fd56-23b5-490d-9af6-d806e4eb17d3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:15.381571645Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1440} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:15.381579905Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1440,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:15.381896895Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1440,"task":"crutest"} 01/30 Write block 47 len 8192 data: 3 3 02/30 Read block 85 len 16384 03/30 Flush 04/30 Write block 249 len 24576 data: 7 6 7 6 6 6 05/30 Write block 151 len 16384 data: 9 8 6 9 06/30 Read block 374 len 12288 07/30 Write block 131 len 24576 data: 3 4 3 3 5 5 08/30 Read block 419 len 36864 09/30 Read block 431 len 16384 10/30 Flush 11/30 Write block 46 len 16384 data: 3 4 4 3 12/30 Write block 407 len 40960 data: 6 7 6 6 3 4 4 5 4 6 13/30 Write block 567 len 40960 data: 6 5 7 8 8 10 7 6 7 8 14/30 Read block 200 len 36864 15/30 Write block 200 len 20480 data: 8 7 7 10 10 16/30 Read block 157 len 24576 17/30 Flush 18/30 Write block 69 len 24576 data: 7 7 6 6 6 4 19/30 Flush 20/30 Write block 379 len 24576 data: 10 9 8 5 6 6 21/30 Write block 247 len 24576 data: 9 9 8 7 8 7 22/30 Read block 165 len 4096 23/30 Flush 24/30 Write block 38 len 32768 data: 6 6 5 6 5 4 3 3 25/30 Write block 457 len 32768 data: 8 10 8 5 5 4 7 6 26/30 Read block 266 len 12288 27/30 Write block 424 len 8192 data: 7 6 28/30 Write block 565 len 28672 data: 5 5 7 6 8 9 9 29/30 Write block 258 len 32768 data: 5 7 7 7 8 10 9 10 30/30 Write block 557 len 4096 data: 7 ---------------------------------------------------------------- Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Sent Done Done false 1001 Acked Read 4 Sent Done Done false 1002 Acked Flush 0 Sent Done Done false 1003 Acked Write 6 Sent Done Done false 1004 Acked Write 4 Sent Done Done false 1005 Acked Read 3 Sent Done Done false 1006 Acked Write 6 Sent Done Done false 1007 Acked Read 9 Sent Done Done false 1008 Acked Read 4 Sent Done Done false 1009 Acked Flush 0 Sent Done Done false 1010 Acked Write 4 Sent Done Done false 1011 Acked Write 10 Sent Done Done false 1012 Acked Write 10 Sent Done Done false 1013 Acked Read 9 Sent Done Done false 1014 Acked Write 5 Sent Done Done false 1015 Acked Read 6 Sent Done Done false 1016 Acked Flush 0 Sent Done Done false 1017 Acked Write 6 Sent Done Done false 1018 Acked Flush 0 Sent Done Done false 1019 Acked Write 6 Sent Done Done false 1020 Acked Write 6 Sent Done Done false 1021 Acked Read 1 Sent Done Done false 1022 Acked Flush 0 Sent Done Done false 1023 Acked Write 8 Sent Done Done false 1024 Acked Write 8 Sent Done Done false 1025 Acked Read 3 Sent Done Done false 1026 Acked Write 2 Sent Done Done false 1027 Acked Write 7 Sent Sent Sent false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 3 3 36 Done 0 27 27 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:22:15.390667178Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1440} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:15.390843689Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1440,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 73 73 73 108 108 108 F F F 1 020-039 68 68 68 99 99 99 F T T <--- 2 040-059 72 76 76 106 114 114 F T T <--- 3 060-079 74 76 76 110 115 115 F F F <--- 4 080-099 75 75 75 111 111 111 F F F 5 100-119 74 74 74 110 110 110 F F F 6 120-139 72 76 76 106 113 113 F F F <--- 7 140-159 75 76 76 111 113 113 F F F <--- 8 160-179 72 72 72 106 106 106 F F F 9 180-199 68 68 68 99 99 99 F F F 10 200-219 67 76 76 98 114 114 F F F <--- 11 220-239 70 70 70 103 103 103 F F F 12 240-259 74 76 76 110 116 116 F T T <--- 13 260-279 75 75 75 111 111 111 F T T <--- 14 280-299 72 72 72 106 106 106 F F F 15 300-319 75 75 75 111 111 111 F F F 16 320-339 75 75 75 111 111 111 F F F 17 340-359 74 74 74 110 110 110 F F F 18 360-379 75 76 76 111 116 116 F F F <--- 19 380-399 75 76 76 111 116 116 F F F <--- 20 400-419 66 76 76 97 114 114 F F F <--- 21 420-439 72 72 72 106 106 106 F T T <--- 22 440-459 68 68 68 99 99 99 F T T <--- 23 460-479 74 74 74 109 109 109 F T T <--- 24 480-499 74 74 74 110 110 110 F F F 25 500-519 71 71 71 104 104 104 F F F 26 520-539 74 74 74 110 110 110 F F F 27 540-559 74 74 74 110 110 110 F T T <--- 28 560-579 70 76 76 100 114 114 F T T <--- 29 580-599 72 72 72 107 107 107 F F F Max gen: 76, Max flush: 116 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 38 Verifying data now No change after verify Loop: 38 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 73 73 73 108 108 108 F F F 1 020-039 77 77 77 117 117 117 F F F 2 040-059 77 77 77 117 117 117 F F F 3 060-079 76 76 76 115 115 115 F F F 4 080-099 75 75 75 111 111 111 F F F 5 100-119 74 74 74 110 110 110 F F F 6 120-139 76 76 76 113 113 113 F F F 7 140-159 76 76 76 113 113 113 F F F 8 160-179 72 72 72 106 106 106 F F F 9 180-199 68 68 68 99 99 99 F F F 10 200-219 76 76 76 114 114 114 F F F 11 220-239 70 70 70 103 103 103 F F F 12 240-259 77 77 77 117 117 117 F F F 13 260-279 77 77 77 117 117 117 F F F 14 280-299 72 72 72 106 106 106 F F F 15 300-319 75 75 75 111 111 111 F F F 16 320-339 75 75 75 111 111 111 F F F 17 340-359 74 74 74 110 110 110 F F F 18 360-379 76 76 76 116 116 116 F F F 19 380-399 76 76 76 116 116 116 F F F 20 400-419 76 76 76 114 114 114 F F F 21 420-439 77 77 77 117 117 117 F F F 22 440-459 77 77 77 117 117 117 F F F 23 460-479 77 77 77 117 117 117 F F F 24 480-499 74 74 74 110 110 110 F F F 25 500-519 71 71 71 104 104 104 F F F 26 520-539 74 74 74 110 110 110 F F F 27 540-559 77 77 77 117 117 117 F F F 28 560-579 77 77 77 117 117 117 F F F 29 580-599 72 72 72 107 107 107 F F F Max gen: 77, Max flush: 117 Begin loop 39 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:19.657125477Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1448,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:19.665387379Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1448,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:19.66568102Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1448} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:19.66571086Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1448} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:19.66571956Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1448} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:19.66573121Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1448} {"msg":"Crucible e6fb6b07-f965-41e6-88e8-5d30b94026e6 has session id: 5d852c47-32f0-40ad-bb8f-c07a79e03f72","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:19.66606903Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1448,"session_id":"5d852c47-32f0-40ad-bb8f-c07a79e03f72"} {"msg":"Upstairs opts: Upstairs UUID: e6fb6b07-f965-41e6-88e8-5d30b94026e6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:19.66609854Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1448,"session_id":"5d852c47-32f0-40ad-bb8f-c07a79e03f72"} {"msg":"Crucible stats registered with UUID: e6fb6b07-f965-41e6-88e8-5d30b94026e6","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:19.66611192Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1448,"session_id":"5d852c47-32f0-40ad-bb8f-c07a79e03f72"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:22:19.6661213Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1448,"session_id":"5d852c47-32f0-40ad-bb8f-c07a79e03f72"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:19.66624262Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1448,"":"io task","client":"0","":"downstairs","session_id":"5d852c47-32f0-40ad-bb8f-c07a79e03f72"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:19.66626034Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1448,"":"io task","client":"1","":"downstairs","session_id":"5d852c47-32f0-40ad-bb8f-c07a79e03f72"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:19.66627053Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1448,"":"io task","client":"2","":"downstairs","session_id":"5d852c47-32f0-40ad-bb8f-c07a79e03f72"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:19.666421631Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1448,"task":"crutest"} {"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:19.666435881Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1448} {"msg":"e6fb6b07-f965-41e6-88e8-5d30b94026e6 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:19.666464121Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1448,"session_id":"5d852c47-32f0-40ad-bb8f-c07a79e03f72"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:19.666473641Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1448,"client":"0","":"downstairs","session_id":"5d852c47-32f0-40ad-bb8f-c07a79e03f72"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:19.666483321Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1448,"client":"1","":"downstairs","session_id":"5d852c47-32f0-40ad-bb8f-c07a79e03f72"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:19.666492941Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1448,"client":"2","":"downstairs","session_id":"5d852c47-32f0-40ad-bb8f-c07a79e03f72"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:19.666502741Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1448,"":"io task","client":"2","":"downstairs","session_id":"5d852c47-32f0-40ad-bb8f-c07a79e03f72"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:19.666512891Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1448,"":"io task","client":"0","":"downstairs","session_id":"5d852c47-32f0-40ad-bb8f-c07a79e03f72"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:19.666522981Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1448,"":"io task","client":"1","":"downstairs","session_id":"5d852c47-32f0-40ad-bb8f-c07a79e03f72"} {"msg":"ds_connection connected from Ok(127.0.0.1:39527)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:19.666581711Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1448,"":"io task","client":"2","":"downstairs","session_id":"5d852c47-32f0-40ad-bb8f-c07a79e03f72"} {"msg":"ds_connection connected from Ok(127.0.0.1:36948)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:19.666605841Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1448,"":"io task","client":"1","":"downstairs","session_id":"5d852c47-32f0-40ad-bb8f-c07a79e03f72"} {"msg":"ds_connection connected from Ok(127.0.0.1:56593)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:19.666618951Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1448,"":"io task","client":"0","":"downstairs","session_id":"5d852c47-32f0-40ad-bb8f-c07a79e03f72"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:19.667233242Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1448,"client":"2","":"downstairs","session_id":"5d852c47-32f0-40ad-bb8f-c07a79e03f72"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:19.667252292Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1448,"client":"1","":"downstairs","session_id":"5d852c47-32f0-40ad-bb8f-c07a79e03f72"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:19.667362692Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1448,"client":"0","":"downstairs","session_id":"5d852c47-32f0-40ad-bb8f-c07a79e03f72"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:19.667578272Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1448,"session_id":"5d852c47-32f0-40ad-bb8f-c07a79e03f72"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:19.667588262Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1448,"session_id":"5d852c47-32f0-40ad-bb8f-c07a79e03f72"} {"msg":"[0]R flush_numbers[0..12]: [108, 117, 117, 115, 111, 110, 113, 113, 106, 99, 114, 103]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:19.667650433Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1448,"":"downstairs","":"downstairs","session_id":"5d852c47-32f0-40ad-bb8f-c07a79e03f72"} {"msg":"[0]R generation[0..12]: [73, 77, 77, 76, 75, 74, 76, 76, 72, 68, 76, 70]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:19.667661663Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1448,"":"downstairs","":"downstairs","session_id":"5d852c47-32f0-40ad-bb8f-c07a79e03f72"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:19.667672843Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1448,"":"downstairs","":"downstairs","session_id":"5d852c47-32f0-40ad-bb8f-c07a79e03f72"} {"msg":"[1]R flush_numbers[0..12]: [108, 117, 117, 115, 111, 110, 113, 113, 106, 99, 114, 103]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:19.667683583Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1448,"":"downstairs","":"downstairs","session_id":"5d852c47-32f0-40ad-bb8f-c07a79e03f72"} {"msg":"[1]R generation[0..12]: [73, 77, 77, 76, 75, 74, 76, 76, 72, 68, 76, 70]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:19.667694063Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1448,"":"downstairs","":"downstairs","session_id":"5d852c47-32f0-40ad-bb8f-c07a79e03f72"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:19.667705073Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1448,"":"downstairs","":"downstairs","session_id":"5d852c47-32f0-40ad-bb8f-c07a79e03f72"} {"msg":"[2]R flush_numbers[0..12]: [108, 117, 117, 115, 111, 110, 113, 113, 106, 99, 114, 103]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:19.667715803Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1448,"":"downstairs","":"downstairs","session_id":"5d852c47-32f0-40ad-bb8f-c07a79e03f72"} {"msg":"[2]R generation[0..12]: [73, 77, 77, 76, 75, 74, 76, 76, 72, 68, 76, 70]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:19.667726403Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1448,"":"downstairs","":"downstairs","session_id":"5d852c47-32f0-40ad-bb8f-c07a79e03f72"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:19.667739623Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1448,"":"downstairs","":"downstairs","session_id":"5d852c47-32f0-40ad-bb8f-c07a79e03f72"} {"msg":"Max found gen is 78","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:19.667749123Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1448,"":"downstairs","":"downstairs","session_id":"5d852c47-32f0-40ad-bb8f-c07a79e03f72"} {"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:19.667758993Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1448,"":"downstairs","":"downstairs","session_id":"5d852c47-32f0-40ad-bb8f-c07a79e03f72"} {"msg":"Next flush: 118","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:19.667768533Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1448,"":"downstairs","":"downstairs","session_id":"5d852c47-32f0-40ad-bb8f-c07a79e03f72"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:19.667778103Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1448,"":"downstairs","":"downstairs","session_id":"5d852c47-32f0-40ad-bb8f-c07a79e03f72"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:19.667788063Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1448,"session_id":"5d852c47-32f0-40ad-bb8f-c07a79e03f72"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:19.667797003Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1448,"client":"0","":"downstairs","session_id":"5d852c47-32f0-40ad-bb8f-c07a79e03f72"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:19.667806883Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1448,"client":"1","":"downstairs","session_id":"5d852c47-32f0-40ad-bb8f-c07a79e03f72"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:19.667816673Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1448,"client":"2","":"downstairs","session_id":"5d852c47-32f0-40ad-bb8f-c07a79e03f72"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:19.667826603Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1448,"session_id":"5d852c47-32f0-40ad-bb8f-c07a79e03f72"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:19.667835853Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1448,"session_id":"5d852c47-32f0-40ad-bb8f-c07a79e03f72"} {"msg":"e6fb6b07-f965-41e6-88e8-5d30b94026e6 is now active with session: 5d852c47-32f0-40ad-bb8f-c07a79e03f72","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:19.667846023Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1448,"session_id":"5d852c47-32f0-40ad-bb8f-c07a79e03f72"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:19.667855003Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1448,"session_id":"5d852c47-32f0-40ad-bb8f-c07a79e03f72"} {"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:19.667864203Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1448} {"msg":"Activated sub_volume e6fb6b07-f965-41e6-88e8-5d30b94026e6","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:19.667872673Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1448} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:19.667881133Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1448,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:19.668181113Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1448,"task":"crutest"} 01/30 Write block 493 len 20480 data: 10 8 6 6 6 02/30 Flush 03/30 Write block 398 len 12288 data: 6 6 4 04/30 Read block 131 len 4096 05/30 Write block 157 len 32768 data: 5 7 6 7 8 5 4 5 06/30 Read block 195 len 16384 07/30 Read block 450 len 8192 08/30 Write block 561 len 28672 data: 7 5 5 5 6 6 8 09/30 Read block 260 len 36864 10/30 Read block 547 len 4096 11/30 Write block 574 len 8192 data: 7 8 12/30 Flush 13/30 Write block 354 len 32768 data: 8 7 6 7 8 9 12 9 14/30 Read block 529 len 24576 15/30 Read block 145 len 36864 16/30 Read block 589 len 40960 17/30 Flush 18/30 Write block 551 len 8192 data: 5 6 19/30 Read block 458 len 36864 20/30 Read block 491 len 16384 21/30 Read block 362 len 24576 22/30 Read block 231 len 24576 23/30 Write block 251 len 4096 data: 9 24/30 Read block 465 len 16384 25/30 Write block 134 len 8192 data: 4 6 26/30 Flush 27/30 Write block 566 len 8192 data: 7 9 28/30 Write block 565 len 12288 data: 7 8 10 29/30 Write block 217 len 12288 data: 6 6 5 30/30 Write block 493 len 24576 data: 11 9 7 7 7 6 ---------------------------------------------------------------- Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Flush 0 Done Sent Done false 1002 Acked Write 3 Done Sent Done false 1003 Acked Read 1 Done Sent Done false 1004 Acked Write 8 Done Sent Done false 1005 Acked Read 4 Done Sent Done false 1006 Acked Read 2 Done Sent Done false 1007 Acked Write 7 Done Sent Done false 1008 Acked Read 9 Done Sent Done false 1009 Acked Read 1 Done Sent Done false 1010 Acked Write 2 Done Sent Done false 1011 Acked Flush 0 Done Sent Done false 1012 Acked Write 8 Done Sent Done false 1013 Acked Read 6 Done Sent Done false 1014 Acked Read 9 Done Sent Done false 1015 Acked Read 10 Done Sent Done false 1016 Acked Flush 0 Done Sent Done false 1017 Acked Write 2 Done Sent Done false 1018 Acked Read 9 Done Sent Done false 1019 Acked Read 4 Done Sent Done false 1020 Acked Read 6 Done Sent Done false 1021 Acked Read 6 Done Sent Done false 1022 Acked Write 1 Done Sent Done false 1023 Acked Read 4 Done Sent Done false 1024 Acked Write 2 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 2 Done Sent Done false 1027 Acked Write 3 Done Sent Sent false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 29 3 34 Done 28 1 27 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) None Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:22:19.676496166Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1448} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:19.676683146Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1448,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 73 73 73 108 108 108 F F F 1 020-039 77 77 77 117 117 117 F F F 2 040-059 77 77 77 117 117 117 F F F 3 060-079 76 76 76 115 115 115 F F F 4 080-099 75 75 75 111 111 111 F F F 5 100-119 74 74 74 110 110 110 F F F 6 120-139 78 76 78 121 113 121 F F F <--- 7 140-159 78 76 78 119 113 119 F F F <--- 8 160-179 78 72 78 119 106 119 F F F <--- 9 180-199 68 68 68 99 99 99 F F F 10 200-219 76 76 76 114 114 114 T F T <--- 11 220-239 70 70 70 103 103 103 F F F 12 240-259 78 77 78 121 117 121 F F F <--- 13 260-279 77 77 77 117 117 117 F F F 14 280-299 72 72 72 106 106 106 F F F 15 300-319 75 75 75 111 111 111 F F F 16 320-339 75 75 75 111 111 111 F F F 17 340-359 78 74 78 120 110 120 F F F <--- 18 360-379 78 76 78 120 116 120 F F F <--- 19 380-399 78 76 78 119 116 119 F F F <--- 20 400-419 78 76 78 119 114 119 F F F <--- 21 420-439 77 77 77 117 117 117 F F F 22 440-459 77 77 77 117 117 117 F F F 23 460-479 77 77 77 117 117 117 F F F 24 480-499 78 74 78 118 110 118 T T T <--- 25 500-519 71 71 71 104 104 104 F F F 26 520-539 74 74 74 110 110 110 F F F 27 540-559 78 77 78 121 117 121 F F F <--- 28 560-579 78 77 78 119 117 119 T F T <--- 29 580-599 72 72 72 107 107 107 F F F Max gen: 78, Max flush: 121 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 39 Verifying data now No change after verify Loop: 39 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 73 73 73 108 108 108 F F F 1 020-039 77 77 77 117 117 117 F F F 2 040-059 77 77 77 117 117 117 F F F 3 060-079 76 76 76 115 115 115 F F F 4 080-099 75 75 75 111 111 111 F F F 5 100-119 74 74 74 110 110 110 F F F 6 120-139 78 78 78 121 121 121 F F F 7 140-159 78 78 78 119 119 119 F F F 8 160-179 78 78 78 119 119 119 F F F 9 180-199 68 68 68 99 99 99 F F F 10 200-219 79 79 79 122 122 122 F F F 11 220-239 70 70 70 103 103 103 F F F 12 240-259 78 78 78 121 121 121 F F F 13 260-279 77 77 77 117 117 117 F F F 14 280-299 72 72 72 106 106 106 F F F 15 300-319 75 75 75 111 111 111 F F F 16 320-339 75 75 75 111 111 111 F F F 17 340-359 78 78 78 120 120 120 F F F 18 360-379 78 78 78 120 120 120 F F F 19 380-399 78 78 78 119 119 119 F F F 20 400-419 78 78 78 119 119 119 F F F 21 420-439 77 77 77 117 117 117 F F F 22 440-459 77 77 77 117 117 117 F F F 23 460-479 77 77 77 117 117 117 F F F 24 480-499 79 79 79 122 122 122 F F F 25 500-519 71 71 71 104 104 104 F F F 26 520-539 74 74 74 110 110 110 F F F 27 540-559 78 78 78 121 121 121 F F F 28 560-579 79 79 79 122 122 122 F F F 29 580-599 72 72 72 107 107 107 F F F Max gen: 79, Max flush: 122 Begin loop 40 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:23.922498383Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1456,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:23.930695035Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1456,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:23.930997186Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1456} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:23.931041956Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1456} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:23.931051156Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1456} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:23.931062946Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1456} {"msg":"Crucible b0fe2cbe-ff8c-486e-a0d2-9ff947827fb9 has session id: 10400191-8133-4eba-bb57-7ae7ce35e5bc","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:23.931376016Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1456,"session_id":"10400191-8133-4eba-bb57-7ae7ce35e5bc"} {"msg":"Upstairs opts: Upstairs UUID: b0fe2cbe-ff8c-486e-a0d2-9ff947827fb9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:23.931428996Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1456,"session_id":"10400191-8133-4eba-bb57-7ae7ce35e5bc"} {"msg":"Crucible stats registered with UUID: b0fe2cbe-ff8c-486e-a0d2-9ff947827fb9","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:23.931439706Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1456,"session_id":"10400191-8133-4eba-bb57-7ae7ce35e5bc"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:22:23.931448736Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1456,"session_id":"10400191-8133-4eba-bb57-7ae7ce35e5bc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:23.931582666Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1456,"":"io task","client":"2","":"downstairs","session_id":"10400191-8133-4eba-bb57-7ae7ce35e5bc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:23.931599996Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1456,"":"io task","client":"0","":"downstairs","session_id":"10400191-8133-4eba-bb57-7ae7ce35e5bc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:23.931610016Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1456,"":"io task","client":"1","":"downstairs","session_id":"10400191-8133-4eba-bb57-7ae7ce35e5bc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:23.931767667Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1456,"task":"crutest"} {"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:23.931786947Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1456} {"msg":"b0fe2cbe-ff8c-486e-a0d2-9ff947827fb9 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:23.931817877Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1456,"session_id":"10400191-8133-4eba-bb57-7ae7ce35e5bc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:23.931827317Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1456,"client":"0","":"downstairs","session_id":"10400191-8133-4eba-bb57-7ae7ce35e5bc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:23.931837087Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1456,"client":"1","":"downstairs","session_id":"10400191-8133-4eba-bb57-7ae7ce35e5bc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:23.931846747Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1456,"client":"2","":"downstairs","session_id":"10400191-8133-4eba-bb57-7ae7ce35e5bc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:23.931856267Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1456,"":"io task","client":"2","":"downstairs","session_id":"10400191-8133-4eba-bb57-7ae7ce35e5bc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:23.931866317Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1456,"":"io task","client":"0","":"downstairs","session_id":"10400191-8133-4eba-bb57-7ae7ce35e5bc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:23.931876277Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1456,"":"io task","client":"1","":"downstairs","session_id":"10400191-8133-4eba-bb57-7ae7ce35e5bc"} {"msg":"ds_connection connected from Ok(127.0.0.1:61879)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:23.931953727Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1456,"":"io task","client":"2","":"downstairs","session_id":"10400191-8133-4eba-bb57-7ae7ce35e5bc"} {"msg":"ds_connection connected from Ok(127.0.0.1:58751)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:23.931979017Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1456,"":"io task","client":"0","":"downstairs","session_id":"10400191-8133-4eba-bb57-7ae7ce35e5bc"} {"msg":"ds_connection connected from Ok(127.0.0.1:50419)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:23.931990597Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1456,"":"io task","client":"1","":"downstairs","session_id":"10400191-8133-4eba-bb57-7ae7ce35e5bc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:23.932649618Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1456,"client":"2","":"downstairs","session_id":"10400191-8133-4eba-bb57-7ae7ce35e5bc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:23.932691088Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1456,"client":"1","":"downstairs","session_id":"10400191-8133-4eba-bb57-7ae7ce35e5bc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:23.932711378Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1456,"client":"0","":"downstairs","session_id":"10400191-8133-4eba-bb57-7ae7ce35e5bc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:23.932964029Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1456,"session_id":"10400191-8133-4eba-bb57-7ae7ce35e5bc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:23.932974009Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1456,"session_id":"10400191-8133-4eba-bb57-7ae7ce35e5bc"} {"msg":"[0]R flush_numbers[0..12]: [108, 117, 117, 115, 111, 110, 121, 119, 119, 99, 122, 103]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:23.933002189Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1456,"":"downstairs","":"downstairs","session_id":"10400191-8133-4eba-bb57-7ae7ce35e5bc"} {"msg":"[0]R generation[0..12]: [73, 77, 77, 76, 75, 74, 78, 78, 78, 68, 79, 70]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:23.933013559Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1456,"":"downstairs","":"downstairs","session_id":"10400191-8133-4eba-bb57-7ae7ce35e5bc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:23.933024589Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1456,"":"downstairs","":"downstairs","session_id":"10400191-8133-4eba-bb57-7ae7ce35e5bc"} {"msg":"[1]R flush_numbers[0..12]: [108, 117, 117, 115, 111, 110, 121, 119, 119, 99, 122, 103]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:23.933035209Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1456,"":"downstairs","":"downstairs","session_id":"10400191-8133-4eba-bb57-7ae7ce35e5bc"} {"msg":"[1]R generation[0..12]: [73, 77, 77, 76, 75, 74, 78, 78, 78, 68, 79, 70]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:23.933045659Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1456,"":"downstairs","":"downstairs","session_id":"10400191-8133-4eba-bb57-7ae7ce35e5bc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:23.933056759Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1456,"":"downstairs","":"downstairs","session_id":"10400191-8133-4eba-bb57-7ae7ce35e5bc"} {"msg":"[2]R flush_numbers[0..12]: [108, 117, 117, 115, 111, 110, 121, 119, 119, 99, 122, 103]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:23.933067519Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1456,"":"downstairs","":"downstairs","session_id":"10400191-8133-4eba-bb57-7ae7ce35e5bc"} {"msg":"[2]R generation[0..12]: [73, 77, 77, 76, 75, 74, 78, 78, 78, 68, 79, 70]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:23.933077939Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1456,"":"downstairs","":"downstairs","session_id":"10400191-8133-4eba-bb57-7ae7ce35e5bc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:23.933089099Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1456,"":"downstairs","":"downstairs","session_id":"10400191-8133-4eba-bb57-7ae7ce35e5bc"} {"msg":"Max found gen is 80","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:23.933098489Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1456,"":"downstairs","":"downstairs","session_id":"10400191-8133-4eba-bb57-7ae7ce35e5bc"} {"msg":"Generation requested: 80 >= found:80","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:23.933108259Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1456,"":"downstairs","":"downstairs","session_id":"10400191-8133-4eba-bb57-7ae7ce35e5bc"} {"msg":"Next flush: 123","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:23.933117609Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1456,"":"downstairs","":"downstairs","session_id":"10400191-8133-4eba-bb57-7ae7ce35e5bc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:23.933127019Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1456,"":"downstairs","":"downstairs","session_id":"10400191-8133-4eba-bb57-7ae7ce35e5bc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:23.933137039Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1456,"session_id":"10400191-8133-4eba-bb57-7ae7ce35e5bc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:23.933147629Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1456,"client":"0","":"downstairs","session_id":"10400191-8133-4eba-bb57-7ae7ce35e5bc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:23.933157679Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1456,"client":"1","":"downstairs","session_id":"10400191-8133-4eba-bb57-7ae7ce35e5bc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:23.933167559Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1456,"client":"2","":"downstairs","session_id":"10400191-8133-4eba-bb57-7ae7ce35e5bc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:23.933177729Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1456,"session_id":"10400191-8133-4eba-bb57-7ae7ce35e5bc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:23.933187089Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1456,"session_id":"10400191-8133-4eba-bb57-7ae7ce35e5bc"} {"msg":"b0fe2cbe-ff8c-486e-a0d2-9ff947827fb9 is now active with session: 10400191-8133-4eba-bb57-7ae7ce35e5bc","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:23.933197049Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1456,"session_id":"10400191-8133-4eba-bb57-7ae7ce35e5bc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:23.933205909Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1456,"session_id":"10400191-8133-4eba-bb57-7ae7ce35e5bc"} {"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:23.933215089Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1456} {"msg":"Activated sub_volume b0fe2cbe-ff8c-486e-a0d2-9ff947827fb9","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:23.933223059Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1456} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:23.933231299Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1456,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:23.933558829Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1456,"task":"crutest"} 01/30 Write block 548 len 20480 data: 5 5 3 6 7 02/30 Write block 330 len 12288 data: 7 8 7 03/30 Write block 511 len 24576 data: 5 6 6 5 3 3 04/30 Read block 34 len 4096 05/30 Read block 188 len 12288 06/30 Read block 298 len 28672 07/30 Write block 355 len 12288 data: 8 7 8 08/30 Write block 578 len 32768 data: 8 5 4 4 5 6 7 7 09/30 Write block 567 len 32768 data: 11 7 9 10 10 11 8 8 10/30 Flush 11/30 Write block 42 len 20480 data: 6 5 4 4 4 12/30 Read block 38 len 24576 13/30 Read block 336 len 8192 14/30 Read block 390 len 24576 15/30 Read block 278 len 8192 16/30 Read block 351 len 20480 17/30 Read block 238 len 8192 18/30 Flush 19/30 Write block 3 len 24576 data: 5 5 4 5 5 5 20/30 Write block 304 len 16384 data: 8 7 7 7 21/30 Write block 400 len 16384 data: 5 3 3 3 22/30 Read block 465 len 24576 23/30 Read block 355 len 24576 24/30 Read block 334 len 20480 25/30 Read block 4 len 28672 26/30 Read block 429 len 20480 27/30 Write block 125 len 36864 data: 6 5 4 5 3 3 4 5 4 28/30 Read block 21 len 20480 29/30 Read block 593 len 12288 30/30 Write block 371 len 8192 data: 7 9 ---------------------------------------------------------------- Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Write 3 Done Sent Done false 1002 Acked Write 6 Done Sent Done false 1003 Acked Read 1 Done Sent Done false 1004 Acked Read 3 Done Sent Done false 1005 Acked Read 7 Done Sent Done false 1006 Acked Write 3 Done Sent Done false 1007 Acked Write 8 Done Sent Done false 1008 Acked Write 8 Done Sent Done false 1009 Acked Flush 0 Done Sent Done false 1010 Acked Write 5 Done Sent Done false 1011 Acked Read 6 Done Sent Done false 1012 Acked Read 2 Done Sent Done false 1013 Acked Read 6 Done Sent Done false 1014 Acked Read 2 Done Sent Done false 1015 Acked Read 5 Done Sent Done false 1016 Acked Read 2 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Write 6 Done Sent Done false 1019 Acked Write 4 Done Sent Done false 1020 Acked Write 4 Done Sent Done false 1021 Acked Read 6 Done Sent Done false 1022 Acked Read 6 Done Sent Done false 1023 Acked Read 5 Done Sent Done false 1024 Acked Read 7 Done Sent Done false 1025 Acked Read 5 Done Sent Done false 1026 Acked Write 9 Done Sent Done false 1027 Acked Read 5 Done Sent Done false 1028 Acked Read 3 Done Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 29 2 32 Done 29 1 28 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) None Some(JobId(1017)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:22:23.941697682Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1456} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:23.941857842Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1456,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 73 73 73 108 108 108 T F T <--- 1 020-039 77 77 77 117 117 117 F F F 2 040-059 80 77 80 124 117 124 F F F <--- 3 060-079 76 76 76 115 115 115 F F F 4 080-099 75 75 75 111 111 111 F F F 5 100-119 74 74 74 110 110 110 F F F 6 120-139 78 78 78 121 121 121 T F T <--- 7 140-159 78 78 78 119 119 119 F F F 8 160-179 78 78 78 119 119 119 F F F 9 180-199 68 68 68 99 99 99 F F F 10 200-219 79 79 79 122 122 122 F F F 11 220-239 70 70 70 103 103 103 F F F 12 240-259 78 78 78 121 121 121 F F F 13 260-279 77 77 77 117 117 117 F F F 14 280-299 72 72 72 106 106 106 F F F 15 300-319 75 75 75 111 111 111 T F T <--- 16 320-339 80 75 80 123 111 123 F F F <--- 17 340-359 80 78 80 123 120 123 F F F <--- 18 360-379 78 78 78 120 120 120 T F T <--- 19 380-399 78 78 78 119 119 119 F F F 20 400-419 78 78 78 119 119 119 T F T <--- 21 420-439 77 77 77 117 117 117 F F F 22 440-459 77 77 77 117 117 117 F F F 23 460-479 77 77 77 117 117 117 F F F 24 480-499 79 79 79 122 122 122 F F F 25 500-519 80 71 80 123 104 123 F F F <--- 26 520-539 74 74 74 110 110 110 F F F 27 540-559 80 78 80 123 121 123 F T F <--- 28 560-579 80 79 80 123 122 123 F F F <--- 29 580-599 80 72 80 123 107 123 F F F <--- Max gen: 80, Max flush: 124 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 40 Verifying data now No change after verify Loop: 40 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 81 81 81 125 125 125 F F F 1 020-039 77 77 77 117 117 117 F F F 2 040-059 80 80 80 124 124 124 F F F 3 060-079 76 76 76 115 115 115 F F F 4 080-099 75 75 75 111 111 111 F F F 5 100-119 74 74 74 110 110 110 F F F 6 120-139 81 81 81 125 125 125 F F F 7 140-159 78 78 78 119 119 119 F F F 8 160-179 78 78 78 119 119 119 F F F 9 180-199 68 68 68 99 99 99 F F F 10 200-219 79 79 79 122 122 122 F F F 11 220-239 70 70 70 103 103 103 F F F 12 240-259 78 78 78 121 121 121 F F F 13 260-279 77 77 77 117 117 117 F F F 14 280-299 72 72 72 106 106 106 F F F 15 300-319 81 81 81 125 125 125 F F F 16 320-339 80 80 80 123 123 123 F F F 17 340-359 80 80 80 123 123 123 F F F 18 360-379 81 81 81 125 125 125 F F F 19 380-399 78 78 78 119 119 119 F F F 20 400-419 81 81 81 125 125 125 F F F 21 420-439 77 77 77 117 117 117 F F F 22 440-459 77 77 77 117 117 117 F F F 23 460-479 77 77 77 117 117 117 F F F 24 480-499 79 79 79 122 122 122 F F F 25 500-519 80 80 80 123 123 123 F F F 26 520-539 74 74 74 110 110 110 F F F 27 540-559 80 80 80 123 123 123 F F F 28 560-579 80 80 80 123 123 123 F F F 29 580-599 80 80 80 123 123 123 F F F Max gen: 81, Max flush: 125 Begin loop 41 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:28.193266317Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1464,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:28.20156577Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1464,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:28.20189047Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1464} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:28.201919451Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1464} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:28.201928501Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1464} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:28.201942851Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1464} {"msg":"Crucible f174f9c2-f46f-4ce6-a12e-365ef091f786 has session id: f010299c-abf8-42a5-8259-df98ad9a9bb8","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:28.202279421Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1464,"session_id":"f010299c-abf8-42a5-8259-df98ad9a9bb8"} {"msg":"Upstairs opts: Upstairs UUID: f174f9c2-f46f-4ce6-a12e-365ef091f786, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:28.202313071Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1464,"session_id":"f010299c-abf8-42a5-8259-df98ad9a9bb8"} {"msg":"Crucible stats registered with UUID: f174f9c2-f46f-4ce6-a12e-365ef091f786","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:28.202324381Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1464,"session_id":"f010299c-abf8-42a5-8259-df98ad9a9bb8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:22:28.202333671Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1464,"session_id":"f010299c-abf8-42a5-8259-df98ad9a9bb8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:28.202482291Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1464,"":"io task","client":"1","":"downstairs","session_id":"f010299c-abf8-42a5-8259-df98ad9a9bb8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:28.202513911Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1464,"":"io task","client":"0","":"downstairs","session_id":"f010299c-abf8-42a5-8259-df98ad9a9bb8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:28.202525041Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1464,"":"io task","client":"2","":"downstairs","session_id":"f010299c-abf8-42a5-8259-df98ad9a9bb8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:28.202651392Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1464,"task":"crutest"} {"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:28.202670572Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1464} {"msg":"f174f9c2-f46f-4ce6-a12e-365ef091f786 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:28.202694442Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1464,"session_id":"f010299c-abf8-42a5-8259-df98ad9a9bb8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:28.202703992Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1464,"client":"0","":"downstairs","session_id":"f010299c-abf8-42a5-8259-df98ad9a9bb8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:28.202713662Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1464,"client":"1","":"downstairs","session_id":"f010299c-abf8-42a5-8259-df98ad9a9bb8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:28.202723422Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1464,"client":"2","":"downstairs","session_id":"f010299c-abf8-42a5-8259-df98ad9a9bb8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:28.202733142Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1464,"":"io task","client":"2","":"downstairs","session_id":"f010299c-abf8-42a5-8259-df98ad9a9bb8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:28.202745172Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1464,"":"io task","client":"0","":"downstairs","session_id":"f010299c-abf8-42a5-8259-df98ad9a9bb8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:28.202755242Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1464,"":"io task","client":"1","":"downstairs","session_id":"f010299c-abf8-42a5-8259-df98ad9a9bb8"} {"msg":"ds_connection connected from Ok(127.0.0.1:37136)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:28.202857042Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1464,"":"io task","client":"2","":"downstairs","session_id":"f010299c-abf8-42a5-8259-df98ad9a9bb8"} {"msg":"ds_connection connected from Ok(127.0.0.1:47973)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:28.202876432Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1464,"":"io task","client":"1","":"downstairs","session_id":"f010299c-abf8-42a5-8259-df98ad9a9bb8"} {"msg":"ds_connection connected from Ok(127.0.0.1:64410)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:28.202954472Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1464,"":"io task","client":"0","":"downstairs","session_id":"f010299c-abf8-42a5-8259-df98ad9a9bb8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:28.203461063Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1464,"client":"0","":"downstairs","session_id":"f010299c-abf8-42a5-8259-df98ad9a9bb8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:28.203530593Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1464,"client":"1","":"downstairs","session_id":"f010299c-abf8-42a5-8259-df98ad9a9bb8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:28.203549193Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1464,"client":"2","":"downstairs","session_id":"f010299c-abf8-42a5-8259-df98ad9a9bb8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:28.203752853Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1464,"session_id":"f010299c-abf8-42a5-8259-df98ad9a9bb8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:28.203769223Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1464,"session_id":"f010299c-abf8-42a5-8259-df98ad9a9bb8"} {"msg":"[0]R flush_numbers[0..12]: [125, 117, 124, 115, 111, 110, 125, 119, 119, 99, 122, 103]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:28.203826203Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1464,"":"downstairs","":"downstairs","session_id":"f010299c-abf8-42a5-8259-df98ad9a9bb8"} {"msg":"[0]R generation[0..12]: [81, 77, 80, 76, 75, 74, 81, 78, 78, 68, 79, 70]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:28.203837743Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1464,"":"downstairs","":"downstairs","session_id":"f010299c-abf8-42a5-8259-df98ad9a9bb8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:28.203849083Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1464,"":"downstairs","":"downstairs","session_id":"f010299c-abf8-42a5-8259-df98ad9a9bb8"} {"msg":"[1]R flush_numbers[0..12]: [125, 117, 124, 115, 111, 110, 125, 119, 119, 99, 122, 103]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:28.203860023Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1464,"":"downstairs","":"downstairs","session_id":"f010299c-abf8-42a5-8259-df98ad9a9bb8"} {"msg":"[1]R generation[0..12]: [81, 77, 80, 76, 75, 74, 81, 78, 78, 68, 79, 70]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:28.203870563Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1464,"":"downstairs","":"downstairs","session_id":"f010299c-abf8-42a5-8259-df98ad9a9bb8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:28.203883813Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1464,"":"downstairs","":"downstairs","session_id":"f010299c-abf8-42a5-8259-df98ad9a9bb8"} {"msg":"[2]R flush_numbers[0..12]: [125, 117, 124, 115, 111, 110, 125, 119, 119, 99, 122, 103]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:28.203894673Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1464,"":"downstairs","":"downstairs","session_id":"f010299c-abf8-42a5-8259-df98ad9a9bb8"} {"msg":"[2]R generation[0..12]: [81, 77, 80, 76, 75, 74, 81, 78, 78, 68, 79, 70]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:28.203905074Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1464,"":"downstairs","":"downstairs","session_id":"f010299c-abf8-42a5-8259-df98ad9a9bb8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:28.203916074Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1464,"":"downstairs","":"downstairs","session_id":"f010299c-abf8-42a5-8259-df98ad9a9bb8"} {"msg":"Max found gen is 82","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:28.203925694Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1464,"":"downstairs","":"downstairs","session_id":"f010299c-abf8-42a5-8259-df98ad9a9bb8"} {"msg":"Generation requested: 82 >= found:82","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:28.203935544Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1464,"":"downstairs","":"downstairs","session_id":"f010299c-abf8-42a5-8259-df98ad9a9bb8"} {"msg":"Next flush: 126","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:28.203944904Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1464,"":"downstairs","":"downstairs","session_id":"f010299c-abf8-42a5-8259-df98ad9a9bb8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:28.203954444Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1464,"":"downstairs","":"downstairs","session_id":"f010299c-abf8-42a5-8259-df98ad9a9bb8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:28.203964644Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1464,"session_id":"f010299c-abf8-42a5-8259-df98ad9a9bb8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:28.203973694Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1464,"client":"0","":"downstairs","session_id":"f010299c-abf8-42a5-8259-df98ad9a9bb8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:28.203983864Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1464,"client":"1","":"downstairs","session_id":"f010299c-abf8-42a5-8259-df98ad9a9bb8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:28.203993934Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1464,"client":"2","":"downstairs","session_id":"f010299c-abf8-42a5-8259-df98ad9a9bb8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:28.204004234Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1464,"session_id":"f010299c-abf8-42a5-8259-df98ad9a9bb8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:28.204013624Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1464,"session_id":"f010299c-abf8-42a5-8259-df98ad9a9bb8"} {"msg":"f174f9c2-f46f-4ce6-a12e-365ef091f786 is now active with session: f010299c-abf8-42a5-8259-df98ad9a9bb8","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:28.204023734Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1464,"session_id":"f010299c-abf8-42a5-8259-df98ad9a9bb8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:28.204033864Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1464,"session_id":"f010299c-abf8-42a5-8259-df98ad9a9bb8"} {"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:28.204043244Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1464} {"msg":"Activated sub_volume f174f9c2-f46f-4ce6-a12e-365ef091f786","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:28.204051354Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1464} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:28.204059584Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1464,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:28.204322784Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1464,"task":"crutest"} 01/30 Write block 167 len 32768 data: 6 7 7 6 6 6 5 4 02/30 Write block 278 len 28672 data: 7 5 7 5 4 3 2 03/30 Read block 524 len 12288 04/30 Write block 491 len 24576 data: 6 9 12 10 8 8 05/30 Read block 451 len 12288 06/30 Write block 238 len 16384 data: 10 13 10 10 07/30 Read block 109 len 24576 08/30 Write block 151 len 40960 data: 10 9 7 10 9 7 6 8 7 8 09/30 Read block 249 len 4096 10/30 Write block 499 len 20480 data: 7 6 6 8 6 11/30 Write block 185 len 12288 data: 7 6 8 12/30 Write block 427 len 32768 data: 5 5 5 4 4 3 4 3 13/30 Write block 52 len 32768 data: 8 8 6 7 6 7 8 7 14/30 Flush 15/30 Write block 404 len 4096 data: 5 16/30 Read block 3 len 16384 17/30 Write block 509 len 28672 data: 5 6 6 7 7 6 4 18/30 Read block 520 len 12288 19/30 Read block 137 len 20480 20/30 Read block 522 len 24576 21/30 Write block 405 len 32768 data: 5 5 7 8 7 7 4 5 22/30 Flush 23/30 Write block 438 len 28672 data: 4 4 3 3 4 4 5 24/30 Read block 132 len 12288 25/30 Read block 183 len 24576 26/30 Write block 363 len 40960 data: 9 11 11 10 7 7 7 7 8 10 27/30 Read block 499 len 4096 28/30 Write block 89 len 40960 data: 8 7 6 6 6 6 8 7 8 7 29/30 Read block 247 len 28672 30/30 Write block 516 len 20480 data: 4 5 6 7 7 ---------------------------------------------------------------- Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Write 7 Done Done Sent false 1002 Acked Read 3 Done Done Sent false 1003 Acked Write 6 Done Done Sent false 1004 Acked Read 3 Done Done Sent false 1005 Acked Write 4 Done Done Sent false 1006 Acked Read 6 Done Done Sent false 1007 Acked Write 10 Done Done Sent false 1008 Acked Read 1 Done Done Sent false 1009 Acked Write 5 Done Done Sent false 1010 Acked Write 3 Done Done Sent false 1011 Acked Write 8 Done Done Sent false 1012 Acked Write 8 Done Done Sent false 1013 Acked Flush 0 Done Done Sent false 1014 Acked Write 1 Done Done Sent false 1015 Acked Read 4 Done Done Sent false 1016 Acked Write 7 Done Done Sent false 1017 Acked Read 3 Done Done Sent false 1018 Acked Read 5 Done Done Sent false 1019 Acked Read 6 Done Done Sent false 1020 Acked Write 8 Done Done Sent false 1021 Acked Flush 0 Done Done Sent false 1022 Acked Write 7 Done Done Sent false 1023 Acked Read 3 Done Done Sent false 1024 Acked Read 6 Done Done Sent false 1025 Acked Write 10 Done Done Sent false 1026 Acked Read 1 Done Done Sent false 1027 Acked Write 10 Done Done Sent false 1028 Acked Read 7 Done Done Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 29 31 Done 29 29 1 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:22:28.213991809Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1464} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:28.214182169Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1464,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 81 81 81 125 125 125 F F F 1 020-039 77 77 77 117 117 117 F F F 2 040-059 82 82 80 126 126 124 F F F <--- 3 060-079 76 76 76 115 115 115 F F F 4 080-099 75 75 75 111 111 111 T T F <--- 5 100-119 74 74 74 110 110 110 F F F 6 120-139 81 81 81 125 125 125 F F F 7 140-159 82 82 78 126 126 119 F F F <--- 8 160-179 82 82 78 126 126 119 F F T <--- 9 180-199 82 82 68 126 126 99 F F F <--- 10 200-219 79 79 79 122 122 122 F F F 11 220-239 82 82 70 126 126 103 F F F <--- 12 240-259 82 82 78 126 126 121 F F F <--- 13 260-279 82 82 77 126 126 117 F F F <--- 14 280-299 82 82 72 126 126 106 F F F <--- 15 300-319 81 81 81 125 125 125 F F F 16 320-339 80 80 80 123 123 123 F F F 17 340-359 80 80 80 123 123 123 F F F 18 360-379 81 81 81 125 125 125 T T F <--- 19 380-399 78 78 78 119 119 119 F F F 20 400-419 82 82 81 127 127 125 F F F <--- 21 420-439 82 82 77 126 126 117 T T F <--- 22 440-459 77 77 77 117 117 117 T T F <--- 23 460-479 77 77 77 117 117 117 F F F 24 480-499 82 82 79 126 126 122 F F F <--- 25 500-519 82 82 80 127 127 123 T T F <--- 26 520-539 74 74 74 110 110 110 T T F <--- 27 540-559 80 80 80 123 123 123 F F F 28 560-579 80 80 80 123 123 123 F F F 29 580-599 80 80 80 123 123 123 F F F Max gen: 82, Max flush: 127 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 41 Verifying data now No change after verify Loop: 41 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 81 81 81 125 125 125 F F F 1 020-039 77 77 77 117 117 117 F F F 2 040-059 82 82 82 126 126 126 F F F 3 060-079 76 76 76 115 115 115 F F F 4 080-099 83 83 83 128 128 128 F F F 5 100-119 74 74 74 110 110 110 F F F 6 120-139 81 81 81 125 125 125 F F F 7 140-159 82 82 82 126 126 126 F F F 8 160-179 82 82 82 126 126 126 F F F 9 180-199 82 82 82 126 126 126 F F F 10 200-219 79 79 79 122 122 122 F F F 11 220-239 82 82 82 126 126 126 F F F 12 240-259 82 82 82 126 126 126 F F F 13 260-279 82 82 82 126 126 126 F F F 14 280-299 82 82 82 126 126 126 F F F 15 300-319 81 81 81 125 125 125 F F F 16 320-339 80 80 80 123 123 123 F F F 17 340-359 80 80 80 123 123 123 F F F 18 360-379 83 83 83 128 128 128 F F F 19 380-399 78 78 78 119 119 119 F F F 20 400-419 82 82 82 127 127 127 F F F 21 420-439 83 83 83 128 128 128 F F F 22 440-459 83 83 83 128 128 128 F F F 23 460-479 77 77 77 117 117 117 F F F 24 480-499 82 82 82 126 126 126 F F F 25 500-519 83 83 83 128 128 128 F F F 26 520-539 83 83 83 128 128 128 F F F 27 540-559 80 80 80 123 123 123 F F F 28 560-579 80 80 80 123 123 123 F F F 29 580-599 80 80 80 123 123 123 F F F Max gen: 83, Max flush: 128 Begin loop 42 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:32.471653874Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1472,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:32.479740276Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1472,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:32.480038636Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1472} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:32.480067486Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1472} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:32.480076206Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1472} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:32.480088006Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1472} {"msg":"Crucible fb2fe9b2-f8db-42f4-854e-8d08dc95b7b2 has session id: e1f97563-bc13-4de5-97ee-6bd17e5b29e9","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:32.480421047Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1472,"session_id":"e1f97563-bc13-4de5-97ee-6bd17e5b29e9"} {"msg":"Upstairs opts: Upstairs UUID: fb2fe9b2-f8db-42f4-854e-8d08dc95b7b2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:32.480452317Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1472,"session_id":"e1f97563-bc13-4de5-97ee-6bd17e5b29e9"} {"msg":"Crucible stats registered with UUID: fb2fe9b2-f8db-42f4-854e-8d08dc95b7b2","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:32.480462897Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1472,"session_id":"e1f97563-bc13-4de5-97ee-6bd17e5b29e9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:22:32.480474227Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1472,"session_id":"e1f97563-bc13-4de5-97ee-6bd17e5b29e9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:32.480625487Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1472,"":"io task","client":"2","":"downstairs","session_id":"e1f97563-bc13-4de5-97ee-6bd17e5b29e9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:32.480643547Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1472,"":"io task","client":"0","":"downstairs","session_id":"e1f97563-bc13-4de5-97ee-6bd17e5b29e9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:32.480653747Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1472,"":"io task","client":"1","":"downstairs","session_id":"e1f97563-bc13-4de5-97ee-6bd17e5b29e9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:32.480774697Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1472,"task":"crutest"} {"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:32.480791327Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1472} {"msg":"fb2fe9b2-f8db-42f4-854e-8d08dc95b7b2 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:32.480829568Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1472,"session_id":"e1f97563-bc13-4de5-97ee-6bd17e5b29e9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:32.480838978Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1472,"client":"0","":"downstairs","session_id":"e1f97563-bc13-4de5-97ee-6bd17e5b29e9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:32.480848868Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1472,"client":"1","":"downstairs","session_id":"e1f97563-bc13-4de5-97ee-6bd17e5b29e9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:32.480858598Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1472,"client":"2","":"downstairs","session_id":"e1f97563-bc13-4de5-97ee-6bd17e5b29e9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:32.480868248Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1472,"":"io task","client":"2","":"downstairs","session_id":"e1f97563-bc13-4de5-97ee-6bd17e5b29e9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:32.480878378Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1472,"":"io task","client":"0","":"downstairs","session_id":"e1f97563-bc13-4de5-97ee-6bd17e5b29e9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:32.480888458Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1472,"":"io task","client":"1","":"downstairs","session_id":"e1f97563-bc13-4de5-97ee-6bd17e5b29e9"} {"msg":"ds_connection connected from Ok(127.0.0.1:41279)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:32.480960258Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1472,"":"io task","client":"1","":"downstairs","session_id":"e1f97563-bc13-4de5-97ee-6bd17e5b29e9"} {"msg":"ds_connection connected from Ok(127.0.0.1:58528)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:32.480985878Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1472,"":"io task","client":"0","":"downstairs","session_id":"e1f97563-bc13-4de5-97ee-6bd17e5b29e9"} {"msg":"ds_connection connected from Ok(127.0.0.1:58681)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:32.480997628Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1472,"":"io task","client":"2","":"downstairs","session_id":"e1f97563-bc13-4de5-97ee-6bd17e5b29e9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:32.481584369Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1472,"client":"0","":"downstairs","session_id":"e1f97563-bc13-4de5-97ee-6bd17e5b29e9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:32.481605559Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1472,"client":"2","":"downstairs","session_id":"e1f97563-bc13-4de5-97ee-6bd17e5b29e9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:32.481656539Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1472,"client":"1","":"downstairs","session_id":"e1f97563-bc13-4de5-97ee-6bd17e5b29e9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:32.481873629Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1472,"session_id":"e1f97563-bc13-4de5-97ee-6bd17e5b29e9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:32.481890929Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1472,"session_id":"e1f97563-bc13-4de5-97ee-6bd17e5b29e9"} {"msg":"[0]R flush_numbers[0..12]: [125, 117, 126, 115, 128, 110, 125, 126, 126, 126, 122, 126]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:32.481946979Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1472,"":"downstairs","":"downstairs","session_id":"e1f97563-bc13-4de5-97ee-6bd17e5b29e9"} {"msg":"[0]R generation[0..12]: [81, 77, 82, 76, 83, 74, 81, 82, 82, 82, 79, 82]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:32.481958109Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1472,"":"downstairs","":"downstairs","session_id":"e1f97563-bc13-4de5-97ee-6bd17e5b29e9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:32.481969259Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1472,"":"downstairs","":"downstairs","session_id":"e1f97563-bc13-4de5-97ee-6bd17e5b29e9"} {"msg":"[1]R flush_numbers[0..12]: [125, 117, 126, 115, 128, 110, 125, 126, 126, 126, 122, 126]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:32.481980029Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1472,"":"downstairs","":"downstairs","session_id":"e1f97563-bc13-4de5-97ee-6bd17e5b29e9"} {"msg":"[1]R generation[0..12]: [81, 77, 82, 76, 83, 74, 81, 82, 82, 82, 79, 82]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:32.481990489Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1472,"":"downstairs","":"downstairs","session_id":"e1f97563-bc13-4de5-97ee-6bd17e5b29e9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:32.482001559Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1472,"":"downstairs","":"downstairs","session_id":"e1f97563-bc13-4de5-97ee-6bd17e5b29e9"} {"msg":"[2]R flush_numbers[0..12]: [125, 117, 126, 115, 128, 110, 125, 126, 126, 126, 122, 126]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:32.482012519Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1472,"":"downstairs","":"downstairs","session_id":"e1f97563-bc13-4de5-97ee-6bd17e5b29e9"} {"msg":"[2]R generation[0..12]: [81, 77, 82, 76, 83, 74, 81, 82, 82, 82, 79, 82]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:32.482023019Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1472,"":"downstairs","":"downstairs","session_id":"e1f97563-bc13-4de5-97ee-6bd17e5b29e9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:32.482034089Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1472,"":"downstairs","":"downstairs","session_id":"e1f97563-bc13-4de5-97ee-6bd17e5b29e9"} {"msg":"Max found gen is 84","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:32.482045939Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1472,"":"downstairs","":"downstairs","session_id":"e1f97563-bc13-4de5-97ee-6bd17e5b29e9"} {"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:32.482055639Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1472,"":"downstairs","":"downstairs","session_id":"e1f97563-bc13-4de5-97ee-6bd17e5b29e9"} {"msg":"Next flush: 129","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:32.482064999Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1472,"":"downstairs","":"downstairs","session_id":"e1f97563-bc13-4de5-97ee-6bd17e5b29e9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:32.482074399Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1472,"":"downstairs","":"downstairs","session_id":"e1f97563-bc13-4de5-97ee-6bd17e5b29e9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:32.482084129Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1472,"session_id":"e1f97563-bc13-4de5-97ee-6bd17e5b29e9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:32.482093009Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1472,"client":"0","":"downstairs","session_id":"e1f97563-bc13-4de5-97ee-6bd17e5b29e9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:32.482102799Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1472,"client":"1","":"downstairs","session_id":"e1f97563-bc13-4de5-97ee-6bd17e5b29e9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:32.482112549Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1472,"client":"2","":"downstairs","session_id":"e1f97563-bc13-4de5-97ee-6bd17e5b29e9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:32.482122439Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1472,"session_id":"e1f97563-bc13-4de5-97ee-6bd17e5b29e9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:32.482131619Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1472,"session_id":"e1f97563-bc13-4de5-97ee-6bd17e5b29e9"} {"msg":"fb2fe9b2-f8db-42f4-854e-8d08dc95b7b2 is now active with session: e1f97563-bc13-4de5-97ee-6bd17e5b29e9","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:32.4821417Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1472,"session_id":"e1f97563-bc13-4de5-97ee-6bd17e5b29e9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:32.4821509Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1472,"session_id":"e1f97563-bc13-4de5-97ee-6bd17e5b29e9"} {"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:32.48216009Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1472} {"msg":"Activated sub_volume fb2fe9b2-f8db-42f4-854e-8d08dc95b7b2","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:32.48216852Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1472} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:32.48217669Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1472,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:32.48249786Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1472,"task":"crutest"} 01/30 Write block 502 len 4096 data: 9 02/30 Write block 467 len 4096 data: 7 03/30 Read block 432 len 28672 04/30 Read block 331 len 32768 05/30 Write block 497 len 40960 data: 8 7 8 7 7 10 7 6 8 8 06/30 Write block 173 len 24576 data: 6 5 4 4 4 7 07/30 Read block 325 len 28672 08/30 Read block 247 len 28672 09/30 Write block 115 len 24576 data: 7 7 5 6 5 5 10/30 Read block 113 len 20480 11/30 Read block 1 len 12288 12/30 Read block 293 len 12288 13/30 Read block 227 len 20480 14/30 Read block 491 len 36864 15/30 Write block 353 len 32768 data: 7 9 9 8 9 9 10 13 16/30 Read block 377 len 12288 17/30 Flush 18/30 Write block 490 len 8192 data: 8 7 19/30 Read block 365 len 36864 20/30 Flush 21/30 Flush 22/30 Write block 495 len 20480 data: 9 9 9 8 9 23/30 Write block 577 len 20480 data: 8 9 6 5 5 24/30 Read block 191 len 16384 25/30 Write block 561 len 12288 data: 8 6 6 26/30 Read block 269 len 16384 27/30 Write block 50 len 8192 data: 6 7 28/30 Write block 217 len 28672 data: 7 7 6 3 3 3 3 29/30 Read block 335 len 20480 30/30 Read block 363 len 40960 ---------------------------------------------------------------- Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Write 1 Done Done Done false 1002 Acked Read 7 Done Done Done false 1003 Acked Read 8 Done Done Done false 1004 Acked Write 10 Done Done Done false 1005 Acked Write 6 Done Done Sent false 1006 Acked Read 7 Done Done Sent false 1007 Acked Read 7 Done Done Sent false 1008 Acked Write 6 Done Done Sent false 1009 Acked Read 5 Done Done Sent false 1010 Acked Read 3 Done Done Sent false 1011 Acked Read 3 Done Done Sent false 1012 Acked Read 5 Done Done Sent false 1013 Acked Read 9 Done Done Sent false 1014 Acked Write 8 Done Done Sent false 1015 Acked Read 3 Done Done Sent false 1016 Acked Flush 0 Done Done Sent false 1017 Acked Write 2 Done Done Sent false 1018 Acked Read 9 Done Done Sent false 1019 Acked Flush 0 Done Done Sent false 1020 Acked Flush 0 Done Done Sent false 1021 Acked Write 5 Done Done Sent false 1022 Acked Write 5 Done Done Sent false 1023 Acked Read 4 Done Done Sent false 1024 Acked Write 3 Done Done Sent false 1025 Acked Read 4 Done Done Sent false 1026 Acked Write 2 Done Done Sent false 1027 Acked Write 7 Done Done Sent false 1028 Acked Read 5 Done Done Sent false 1029 Acked Read 10 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 25 25 Done 30 30 5 65 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:22:32.491247183Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1472} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:32.491433444Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1472,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 81 81 81 125 125 125 F F F 1 020-039 77 77 77 117 117 117 F F F 2 040-059 82 82 82 126 126 126 T T F <--- 3 060-079 76 76 76 115 115 115 F F F 4 080-099 83 83 83 128 128 128 F F F 5 100-119 84 84 74 129 129 110 F F F <--- 6 120-139 84 84 81 129 129 125 F F F <--- 7 140-159 82 82 82 126 126 126 F F F 8 160-179 84 84 82 129 129 126 F F F <--- 9 180-199 82 82 82 126 126 126 F F F 10 200-219 79 79 79 122 122 122 T T F <--- 11 220-239 82 82 82 126 126 126 T T F <--- 12 240-259 82 82 82 126 126 126 F F F 13 260-279 82 82 82 126 126 126 F F F 14 280-299 82 82 82 126 126 126 F F F 15 300-319 81 81 81 125 125 125 F F F 16 320-339 80 80 80 123 123 123 F F F 17 340-359 84 84 80 129 129 123 F F F <--- 18 360-379 84 84 83 129 129 128 F F F <--- 19 380-399 78 78 78 119 119 119 F F F 20 400-419 82 82 82 127 127 127 F F F 21 420-439 83 83 83 128 128 128 F F F 22 440-459 83 83 83 128 128 128 F F F 23 460-479 84 84 77 129 129 117 F F T <--- 24 480-499 84 84 82 130 130 126 T T T <--- 25 500-519 84 84 83 129 129 128 F F T <--- 26 520-539 83 83 83 128 128 128 F F F 27 540-559 80 80 80 123 123 123 F F F 28 560-579 80 80 80 123 123 123 T T F <--- 29 580-599 80 80 80 123 123 123 T T F <--- Max gen: 84, Max flush: 130 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 42 Verifying data now No change after verify Loop: 42 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 81 81 81 125 125 125 F F F 1 020-039 77 77 77 117 117 117 F F F 2 040-059 85 85 85 131 131 131 F F F 3 060-079 76 76 76 115 115 115 F F F 4 080-099 83 83 83 128 128 128 F F F 5 100-119 84 84 84 129 129 129 F F F 6 120-139 84 84 84 129 129 129 F F F 7 140-159 82 82 82 126 126 126 F F F 8 160-179 84 84 84 129 129 129 F F F 9 180-199 82 82 82 126 126 126 F F F 10 200-219 85 85 85 131 131 131 F F F 11 220-239 85 85 85 131 131 131 F F F 12 240-259 82 82 82 126 126 126 F F F 13 260-279 82 82 82 126 126 126 F F F 14 280-299 82 82 82 126 126 126 F F F 15 300-319 81 81 81 125 125 125 F F F 16 320-339 80 80 80 123 123 123 F F F 17 340-359 84 84 84 129 129 129 F F F 18 360-379 84 84 84 129 129 129 F F F 19 380-399 78 78 78 119 119 119 F F F 20 400-419 82 82 82 127 127 127 F F F 21 420-439 83 83 83 128 128 128 F F F 22 440-459 83 83 83 128 128 128 F F F 23 460-479 84 84 84 129 129 129 F F F 24 480-499 85 85 85 131 131 131 F F F 25 500-519 84 84 84 129 129 129 F F F 26 520-539 83 83 83 128 128 128 F F F 27 540-559 80 80 80 123 123 123 F F F 28 560-579 85 85 85 131 131 131 F F F 29 580-599 85 85 85 131 131 131 F F F Max gen: 85, Max flush: 131 Begin loop 43 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:36.746770685Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1480,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:36.754943857Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1480,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:36.755240908Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1480} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:36.755268438Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1480} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:36.755277308Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1480} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:36.755289308Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1480} {"msg":"Crucible 88cbd810-591b-46ea-8d2a-d8766ac14454 has session id: 38c87608-a6da-4175-ab9c-d296162819c3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:36.755646028Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1480,"session_id":"38c87608-a6da-4175-ab9c-d296162819c3"} {"msg":"Upstairs opts: Upstairs UUID: 88cbd810-591b-46ea-8d2a-d8766ac14454, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:36.755674988Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1480,"session_id":"38c87608-a6da-4175-ab9c-d296162819c3"} {"msg":"Crucible stats registered with UUID: 88cbd810-591b-46ea-8d2a-d8766ac14454","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:36.755686008Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1480,"session_id":"38c87608-a6da-4175-ab9c-d296162819c3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:22:36.755695258Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1480,"session_id":"38c87608-a6da-4175-ab9c-d296162819c3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:36.755823429Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1480,"":"io task","client":"2","":"downstairs","session_id":"38c87608-a6da-4175-ab9c-d296162819c3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:36.755854199Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1480,"":"io task","client":"0","":"downstairs","session_id":"38c87608-a6da-4175-ab9c-d296162819c3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:36.755865399Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1480,"":"io task","client":"1","":"downstairs","session_id":"38c87608-a6da-4175-ab9c-d296162819c3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:36.755995219Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1480,"task":"crutest"} {"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:36.756014149Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1480} {"msg":"88cbd810-591b-46ea-8d2a-d8766ac14454 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:36.756044399Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1480,"session_id":"38c87608-a6da-4175-ab9c-d296162819c3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:36.756053939Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1480,"client":"0","":"downstairs","session_id":"38c87608-a6da-4175-ab9c-d296162819c3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:36.756063739Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1480,"client":"1","":"downstairs","session_id":"38c87608-a6da-4175-ab9c-d296162819c3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:36.756073429Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1480,"client":"2","":"downstairs","session_id":"38c87608-a6da-4175-ab9c-d296162819c3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:36.756083029Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1480,"":"io task","client":"2","":"downstairs","session_id":"38c87608-a6da-4175-ab9c-d296162819c3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:36.756093089Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1480,"":"io task","client":"0","":"downstairs","session_id":"38c87608-a6da-4175-ab9c-d296162819c3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:36.756103069Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1480,"":"io task","client":"1","":"downstairs","session_id":"38c87608-a6da-4175-ab9c-d296162819c3"} {"msg":"ds_connection connected from Ok(127.0.0.1:32939)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:36.756253069Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1480,"":"io task","client":"2","":"downstairs","session_id":"38c87608-a6da-4175-ab9c-d296162819c3"} {"msg":"ds_connection connected from Ok(127.0.0.1:39843)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:36.756336169Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1480,"":"io task","client":"1","":"downstairs","session_id":"38c87608-a6da-4175-ab9c-d296162819c3"} {"msg":"ds_connection connected from Ok(127.0.0.1:43004)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:36.756350549Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1480,"":"io task","client":"0","":"downstairs","session_id":"38c87608-a6da-4175-ab9c-d296162819c3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:36.75692388Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1480,"client":"0","":"downstairs","session_id":"38c87608-a6da-4175-ab9c-d296162819c3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:36.75694624Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1480,"client":"1","":"downstairs","session_id":"38c87608-a6da-4175-ab9c-d296162819c3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:36.75697605Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1480,"client":"2","":"downstairs","session_id":"38c87608-a6da-4175-ab9c-d296162819c3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:36.757261461Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1480,"session_id":"38c87608-a6da-4175-ab9c-d296162819c3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:36.757273461Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1480,"session_id":"38c87608-a6da-4175-ab9c-d296162819c3"} {"msg":"[0]R flush_numbers[0..12]: [125, 117, 131, 115, 128, 129, 129, 126, 129, 126, 131, 131]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:36.757316091Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1480,"":"downstairs","":"downstairs","session_id":"38c87608-a6da-4175-ab9c-d296162819c3"} {"msg":"[0]R generation[0..12]: [81, 77, 85, 76, 83, 84, 84, 82, 84, 82, 85, 85]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:36.757327481Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1480,"":"downstairs","":"downstairs","session_id":"38c87608-a6da-4175-ab9c-d296162819c3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:36.757338781Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1480,"":"downstairs","":"downstairs","session_id":"38c87608-a6da-4175-ab9c-d296162819c3"} {"msg":"[1]R flush_numbers[0..12]: [125, 117, 131, 115, 128, 129, 129, 126, 129, 126, 131, 131]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:36.757349681Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1480,"":"downstairs","":"downstairs","session_id":"38c87608-a6da-4175-ab9c-d296162819c3"} {"msg":"[1]R generation[0..12]: [81, 77, 85, 76, 83, 84, 84, 82, 84, 82, 85, 85]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:36.757360051Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1480,"":"downstairs","":"downstairs","session_id":"38c87608-a6da-4175-ab9c-d296162819c3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:36.757371081Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1480,"":"downstairs","":"downstairs","session_id":"38c87608-a6da-4175-ab9c-d296162819c3"} {"msg":"[2]R flush_numbers[0..12]: [125, 117, 131, 115, 128, 129, 129, 126, 129, 126, 131, 131]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:36.757381951Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1480,"":"downstairs","":"downstairs","session_id":"38c87608-a6da-4175-ab9c-d296162819c3"} {"msg":"[2]R generation[0..12]: [81, 77, 85, 76, 83, 84, 84, 82, 84, 82, 85, 85]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:36.757392441Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1480,"":"downstairs","":"downstairs","session_id":"38c87608-a6da-4175-ab9c-d296162819c3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:36.757403531Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1480,"":"downstairs","":"downstairs","session_id":"38c87608-a6da-4175-ab9c-d296162819c3"} {"msg":"Max found gen is 86","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:36.757413151Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1480,"":"downstairs","":"downstairs","session_id":"38c87608-a6da-4175-ab9c-d296162819c3"} {"msg":"Generation requested: 86 >= found:86","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:36.757422831Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1480,"":"downstairs","":"downstairs","session_id":"38c87608-a6da-4175-ab9c-d296162819c3"} {"msg":"Next flush: 132","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:36.757432281Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1480,"":"downstairs","":"downstairs","session_id":"38c87608-a6da-4175-ab9c-d296162819c3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:36.757441791Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1480,"":"downstairs","":"downstairs","session_id":"38c87608-a6da-4175-ab9c-d296162819c3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:36.757451691Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1480,"session_id":"38c87608-a6da-4175-ab9c-d296162819c3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:36.757460751Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1480,"client":"0","":"downstairs","session_id":"38c87608-a6da-4175-ab9c-d296162819c3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:36.757472061Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1480,"client":"1","":"downstairs","session_id":"38c87608-a6da-4175-ab9c-d296162819c3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:36.757481701Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1480,"client":"2","":"downstairs","session_id":"38c87608-a6da-4175-ab9c-d296162819c3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:36.757491631Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1480,"session_id":"38c87608-a6da-4175-ab9c-d296162819c3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:36.757500801Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1480,"session_id":"38c87608-a6da-4175-ab9c-d296162819c3"} {"msg":"88cbd810-591b-46ea-8d2a-d8766ac14454 is now active with session: 38c87608-a6da-4175-ab9c-d296162819c3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:36.757510901Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1480,"session_id":"38c87608-a6da-4175-ab9c-d296162819c3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:36.757519801Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1480,"session_id":"38c87608-a6da-4175-ab9c-d296162819c3"} {"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:36.757528971Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1480} {"msg":"Activated sub_volume 88cbd810-591b-46ea-8d2a-d8766ac14454","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:36.757537021Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1480} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:36.757545341Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1480,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:36.757854142Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1480,"task":"crutest"} 01/30 Write block 485 len 24576 data: 9 9 10 8 10 9 02/30 Write block 79 len 12288 data: 6 6 5 03/30 Write block 61 len 16384 data: 10 9 8 8 04/30 Read block 410 len 24576 05/30 Read block 521 len 24576 06/30 Read block 70 len 28672 07/30 Read block 494 len 40960 08/30 Read block 190 len 36864 09/30 Read block 438 len 40960 10/30 Flush 11/30 Write block 319 len 20480 data: 7 9 8 10 10 12/30 Flush 13/30 Write block 292 len 40960 data: 6 9 8 9 9 7 9 10 9 11 14/30 Write block 76 len 40960 data: 4 5 6 7 7 6 6 6 6 8 15/30 Read block 427 len 24576 16/30 Read block 317 len 24576 17/30 Read block 0 len 28672 18/30 Flush 19/30 Write block 381 len 36864 data: 9 6 7 7 8 8 7 9 9 20/30 Read block 434 len 12288 21/30 Read block 464 len 4096 22/30 Read block 408 len 16384 23/30 Read block 403 len 40960 24/30 Write block 490 len 4096 data: 10 25/30 Read block 546 len 28672 26/30 Read block 376 len 40960 27/30 Read block 418 len 16384 28/30 Write block 320 len 16384 data: 10 9 11 11 29/30 Read block 78 len 16384 30/30 Read block 522 len 12288 ---------------------------------------------------------------- Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Write 3 Done Done Done false 1002 Acked Write 4 Done Done Done false 1003 Acked Read 6 Done Done Done false 1004 Acked Read 6 Done Done Done false 1005 Acked Read 7 Done Done Done false 1006 Acked Read 10 Done Done Done false 1007 Acked Read 9 Done Done Done false 1008 Acked Read 10 Done Sent Done false 1009 Acked Flush 0 Done Sent Done false 1010 Acked Write 5 Done Sent Done false 1011 Acked Flush 0 Done Sent Done false 1012 Acked Write 10 Done Sent Done false 1013 Acked Write 10 Done Sent Done false 1014 Acked Read 6 Done Sent Done false 1015 Acked Read 6 Done Sent Done false 1016 Acked Read 7 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Write 9 Done Sent Done false 1019 Acked Read 3 Done Sent Done false 1020 Acked Read 1 Done Sent Done false 1021 Acked Read 4 Done Sent Done false 1022 Acked Read 10 Done Sent Done false 1023 Acked Write 1 Done Sent Done false 1024 Acked Read 7 Done Sent Done false 1025 Acked Read 10 Done Sent Done false 1026 Acked Read 4 Done Sent Done false 1027 Acked Write 4 Done Sent Done false 1028 Acked Read 4 Done Sent Done false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 22 1 23 Done 30 8 29 67 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) None Some(JobId(1017)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:22:36.766958405Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1480} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:36.767149476Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1480,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 81 81 81 125 125 125 F F F 1 020-039 77 77 77 117 117 117 F F F 2 040-059 85 85 85 131 131 131 F F F 3 060-079 86 76 86 134 115 134 F T F <--- 4 080-099 86 83 86 134 128 134 F T F <--- 5 100-119 84 84 84 129 129 129 F F F 6 120-139 84 84 84 129 129 129 F F F 7 140-159 82 82 82 126 126 126 F F F 8 160-179 84 84 84 129 129 129 F F F 9 180-199 82 82 82 126 126 126 F F F 10 200-219 85 85 85 131 131 131 F F F 11 220-239 85 85 85 131 131 131 F F F 12 240-259 82 82 82 126 126 126 F F F 13 260-279 82 82 82 126 126 126 F F F 14 280-299 86 82 86 134 126 134 F F F <--- 15 300-319 86 81 86 134 125 134 F F F <--- 16 320-339 86 80 86 133 123 133 T F T <--- 17 340-359 84 84 84 129 129 129 F F F 18 360-379 84 84 84 129 129 129 F F F 19 380-399 78 78 78 119 119 119 T F T <--- 20 400-419 82 82 82 127 127 127 F F F 21 420-439 83 83 83 128 128 128 F F F 22 440-459 83 83 83 128 128 128 F F F 23 460-479 84 84 84 129 129 129 F F F 24 480-499 86 85 86 132 131 132 T T T <--- 25 500-519 84 84 84 129 129 129 F F F 26 520-539 83 83 83 128 128 128 F F F 27 540-559 80 80 80 123 123 123 F F F 28 560-579 85 85 85 131 131 131 F F F 29 580-599 85 85 85 131 131 131 F F F Max gen: 86, Max flush: 134 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 43 Verifying data now No change after verify Loop: 43 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 81 81 81 125 125 125 F F F 1 020-039 77 77 77 117 117 117 F F F 2 040-059 85 85 85 131 131 131 F F F 3 060-079 86 86 86 134 134 134 F F F 4 080-099 86 86 86 134 134 134 F F F 5 100-119 84 84 84 129 129 129 F F F 6 120-139 84 84 84 129 129 129 F F F 7 140-159 82 82 82 126 126 126 F F F 8 160-179 84 84 84 129 129 129 F F F 9 180-199 82 82 82 126 126 126 F F F 10 200-219 85 85 85 131 131 131 F F F 11 220-239 85 85 85 131 131 131 F F F 12 240-259 82 82 82 126 126 126 F F F 13 260-279 82 82 82 126 126 126 F F F 14 280-299 86 86 86 134 134 134 F F F 15 300-319 86 86 86 134 134 134 F F F 16 320-339 87 87 87 135 135 135 F F F 17 340-359 84 84 84 129 129 129 F F F 18 360-379 84 84 84 129 129 129 F F F 19 380-399 87 87 87 135 135 135 F F F 20 400-419 82 82 82 127 127 127 F F F 21 420-439 83 83 83 128 128 128 F F F 22 440-459 83 83 83 128 128 128 F F F 23 460-479 84 84 84 129 129 129 F F F 24 480-499 87 87 87 135 135 135 F F F 25 500-519 84 84 84 129 129 129 F F F 26 520-539 83 83 83 128 128 128 F F F 27 540-559 80 80 80 123 123 123 F F F 28 560-579 85 85 85 131 131 131 F F F 29 580-599 85 85 85 131 131 131 F F F Max gen: 87, Max flush: 135 Begin loop 44 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:41.00506606Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1488,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:22:41.012217121Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1488,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:41.012271201Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1488,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:41.019323202Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1488,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:41.019599272Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1488} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:41.019627642Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1488} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:41.019638932Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1488} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:41.019650682Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1488} {"msg":"Crucible 783d577d-ba12-4cf0-9d36-d28c468fcdd1 has session id: 80a46d75-6e0c-4748-99be-6b346fa3edb7","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:41.019954843Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1488,"session_id":"80a46d75-6e0c-4748-99be-6b346fa3edb7"} {"msg":"Upstairs opts: Upstairs UUID: 783d577d-ba12-4cf0-9d36-d28c468fcdd1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:41.019975363Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1488,"session_id":"80a46d75-6e0c-4748-99be-6b346fa3edb7"} {"msg":"Crucible stats registered with UUID: 783d577d-ba12-4cf0-9d36-d28c468fcdd1","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:41.019985443Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1488,"session_id":"80a46d75-6e0c-4748-99be-6b346fa3edb7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:22:41.019994413Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1488,"session_id":"80a46d75-6e0c-4748-99be-6b346fa3edb7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:41.020142173Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1488,"":"io task","client":"0","":"downstairs","session_id":"80a46d75-6e0c-4748-99be-6b346fa3edb7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:41.020159763Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1488,"":"io task","client":"2","":"downstairs","session_id":"80a46d75-6e0c-4748-99be-6b346fa3edb7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:41.020169873Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1488,"":"io task","client":"1","":"downstairs","session_id":"80a46d75-6e0c-4748-99be-6b346fa3edb7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:41.020323363Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1488,"task":"crutest"} {"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:41.020339903Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1488} {"msg":"783d577d-ba12-4cf0-9d36-d28c468fcdd1 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:41.020371583Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1488,"session_id":"80a46d75-6e0c-4748-99be-6b346fa3edb7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:41.020380993Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1488,"client":"0","":"downstairs","session_id":"80a46d75-6e0c-4748-99be-6b346fa3edb7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:41.020390743Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1488,"client":"1","":"downstairs","session_id":"80a46d75-6e0c-4748-99be-6b346fa3edb7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:41.020400424Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1488,"client":"2","":"downstairs","session_id":"80a46d75-6e0c-4748-99be-6b346fa3edb7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:41.020412424Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1488,"":"io task","client":"2","":"downstairs","session_id":"80a46d75-6e0c-4748-99be-6b346fa3edb7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:41.020422434Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1488,"":"io task","client":"0","":"downstairs","session_id":"80a46d75-6e0c-4748-99be-6b346fa3edb7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:41.020432494Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1488,"":"io task","client":"1","":"downstairs","session_id":"80a46d75-6e0c-4748-99be-6b346fa3edb7"} {"msg":"ds_connection connected from Ok(127.0.0.1:35820)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:41.020479544Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1488,"":"io task","client":"0","":"downstairs","session_id":"80a46d75-6e0c-4748-99be-6b346fa3edb7"} {"msg":"ds_connection connected from Ok(127.0.0.1:51880)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:41.020490454Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1488,"":"io task","client":"1","":"downstairs","session_id":"80a46d75-6e0c-4748-99be-6b346fa3edb7"} {"msg":"ds_connection connected from Ok(127.0.0.1:57509)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:41.020500964Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1488,"":"io task","client":"2","":"downstairs","session_id":"80a46d75-6e0c-4748-99be-6b346fa3edb7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:41.021167135Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1488,"client":"1","":"downstairs","session_id":"80a46d75-6e0c-4748-99be-6b346fa3edb7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:41.021212885Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1488,"client":"2","":"downstairs","session_id":"80a46d75-6e0c-4748-99be-6b346fa3edb7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:41.021224845Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1488,"client":"0","":"downstairs","session_id":"80a46d75-6e0c-4748-99be-6b346fa3edb7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:41.021485195Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1488,"session_id":"80a46d75-6e0c-4748-99be-6b346fa3edb7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:41.021522865Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1488,"session_id":"80a46d75-6e0c-4748-99be-6b346fa3edb7"} {"msg":"[0]R flush_numbers[0..12]: [125, 117, 131, 134, 134, 129, 129, 126, 129, 126, 131, 131]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:41.021561375Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1488,"":"downstairs","":"downstairs","session_id":"80a46d75-6e0c-4748-99be-6b346fa3edb7"} {"msg":"[0]R generation[0..12]: [81, 77, 85, 86, 86, 84, 84, 82, 84, 82, 85, 85]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:41.021572605Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1488,"":"downstairs","":"downstairs","session_id":"80a46d75-6e0c-4748-99be-6b346fa3edb7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:41.021583905Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1488,"":"downstairs","":"downstairs","session_id":"80a46d75-6e0c-4748-99be-6b346fa3edb7"} {"msg":"[1]R flush_numbers[0..12]: [125, 117, 131, 134, 134, 129, 129, 126, 129, 126, 131, 131]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:41.021594585Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1488,"":"downstairs","":"downstairs","session_id":"80a46d75-6e0c-4748-99be-6b346fa3edb7"} {"msg":"[1]R generation[0..12]: [81, 77, 85, 86, 86, 84, 84, 82, 84, 82, 85, 85]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:41.021607145Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1488,"":"downstairs","":"downstairs","session_id":"80a46d75-6e0c-4748-99be-6b346fa3edb7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:41.021618235Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1488,"":"downstairs","":"downstairs","session_id":"80a46d75-6e0c-4748-99be-6b346fa3edb7"} {"msg":"[2]R flush_numbers[0..12]: [125, 117, 131, 134, 134, 129, 129, 126, 129, 126, 131, 131]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:41.021628945Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1488,"":"downstairs","":"downstairs","session_id":"80a46d75-6e0c-4748-99be-6b346fa3edb7"} {"msg":"[2]R generation[0..12]: [81, 77, 85, 86, 86, 84, 84, 82, 84, 82, 85, 85]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:41.021639325Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1488,"":"downstairs","":"downstairs","session_id":"80a46d75-6e0c-4748-99be-6b346fa3edb7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:41.021650485Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1488,"":"downstairs","":"downstairs","session_id":"80a46d75-6e0c-4748-99be-6b346fa3edb7"} {"msg":"Max found gen is 88","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:41.021659995Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1488,"":"downstairs","":"downstairs","session_id":"80a46d75-6e0c-4748-99be-6b346fa3edb7"} {"msg":"Generation requested: 88 >= found:88","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:41.021669835Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1488,"":"downstairs","":"downstairs","session_id":"80a46d75-6e0c-4748-99be-6b346fa3edb7"} {"msg":"Next flush: 136","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:41.021679205Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1488,"":"downstairs","":"downstairs","session_id":"80a46d75-6e0c-4748-99be-6b346fa3edb7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:41.021688605Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1488,"":"downstairs","":"downstairs","session_id":"80a46d75-6e0c-4748-99be-6b346fa3edb7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:41.021698645Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1488,"session_id":"80a46d75-6e0c-4748-99be-6b346fa3edb7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:41.021707385Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1488,"client":"0","":"downstairs","session_id":"80a46d75-6e0c-4748-99be-6b346fa3edb7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:41.021717046Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1488,"client":"1","":"downstairs","session_id":"80a46d75-6e0c-4748-99be-6b346fa3edb7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:41.021726856Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1488,"client":"2","":"downstairs","session_id":"80a46d75-6e0c-4748-99be-6b346fa3edb7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:41.021736806Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1488,"session_id":"80a46d75-6e0c-4748-99be-6b346fa3edb7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:41.021746116Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1488,"session_id":"80a46d75-6e0c-4748-99be-6b346fa3edb7"} {"msg":"783d577d-ba12-4cf0-9d36-d28c468fcdd1 is now active with session: 80a46d75-6e0c-4748-99be-6b346fa3edb7","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:41.021757296Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1488,"session_id":"80a46d75-6e0c-4748-99be-6b346fa3edb7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:41.021766186Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1488,"session_id":"80a46d75-6e0c-4748-99be-6b346fa3edb7"} {"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:41.021775546Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1488} {"msg":"Activated sub_volume 783d577d-ba12-4cf0-9d36-d28c468fcdd1","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:41.021783886Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1488} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:41.021792246Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1488,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:41.022092916Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1488,"task":"crutest"} 01/30 Write block 189 len 20480 data: 7 7 9 11 10 02/30 Write block 100 len 8192 data: 5 3 03/30 Read block 400 len 40960 04/30 Read block 259 len 12288 05/30 Read block 460 len 8192 06/30 Read block 262 len 24576 07/30 Read block 541 len 12288 08/30 Read block 91 len 24576 09/30 Read block 66 len 4096 10/30 Read block 361 len 40960 11/30 Flush 12/30 Write block 532 len 40960 data: 6 6 5 6 5 6 7 8 7 7 13/30 Read block 422 len 16384 14/30 Write block 156 len 12288 data: 8 7 9 15/30 Read block 357 len 36864 16/30 Read block 84 len 36864 17/30 Read block 208 len 28672 18/30 Read block 378 len 20480 19/30 Read block 126 len 28672 20/30 Write block 62 len 8192 data: 10 9 21/30 Flush 22/30 Write block 486 len 12288 data: 10 11 9 23/30 Read block 5 len 20480 24/30 Write block 532 len 28672 data: 7 7 6 7 6 7 8 25/30 Read block 292 len 40960 26/30 Read block 115 len 28672 27/30 Write block 503 len 16384 data: 8 7 9 9 28/30 Read block 89 len 36864 29/30 Write block 573 len 28672 data: 9 9 9 9 9 10 7 30/30 Read block 101 len 20480 ---------------------------------------------------------------- Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Write 2 Sent Done Done false 1002 Acked Read 10 Sent Done Done false 1003 Acked Read 3 Sent Done Done false 1004 Acked Read 2 Sent Done Done false 1005 Acked Read 6 Sent Done Done false 1006 Acked Read 3 Sent Done Done false 1007 Acked Read 6 Sent Done Done false 1008 Acked Read 1 Sent Done Done false 1009 Acked Read 10 Sent Done Done false 1010 Acked Flush 0 Sent Done Done false 1011 Acked Write 10 Sent Done Done false 1012 Acked Read 4 Sent Done Done false 1013 Acked Write 3 Sent Done Done false 1014 Acked Read 9 Sent Done Done false 1015 Acked Read 9 Sent Done Done false 1016 Acked Read 7 Sent Done Done false 1017 Acked Read 5 Sent Done Done false 1018 Acked Read 7 Sent Done Done false 1019 Acked Write 2 Sent Done Done false 1020 Acked Flush 0 Sent Done Done false 1021 Acked Write 3 Sent Done Done false 1022 Acked Read 5 Sent Done Done false 1023 Acked Write 7 Sent Done Done false 1024 Acked Read 10 Sent Done Done false 1025 Acked Read 7 Sent Done Done false 1026 Acked Write 4 Sent Done Done false 1027 Acked Read 9 Sent Done Done false 1028 Acked Write 7 Sent Done Done false 1029 Acked Read 5 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 1 0 30 Done 1 29 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:22:41.03139169Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1488} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:41.03157721Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1488,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 81 81 81 125 125 125 F F F 1 020-039 77 77 77 117 117 117 F F F 2 040-059 85 85 85 131 131 131 F F F 3 060-079 86 88 88 134 137 137 F F F <--- 4 080-099 86 86 86 134 134 134 F F F 5 100-119 84 88 88 129 136 136 F F F <--- 6 120-139 84 84 84 129 129 129 F F F 7 140-159 82 88 88 126 137 137 F F F <--- 8 160-179 84 84 84 129 129 129 F F F 9 180-199 82 88 88 126 136 136 T F F <--- 10 200-219 85 85 85 131 131 131 F F F 11 220-239 85 85 85 131 131 131 F F F 12 240-259 82 82 82 126 126 126 F F F 13 260-279 82 82 82 126 126 126 F F F 14 280-299 86 86 86 134 134 134 F F F 15 300-319 86 86 86 134 134 134 F F F 16 320-339 87 87 87 135 135 135 F F F 17 340-359 84 84 84 129 129 129 F F F 18 360-379 84 84 84 129 129 129 F F F 19 380-399 87 87 87 135 135 135 F F F 20 400-419 82 82 82 127 127 127 F F F 21 420-439 83 83 83 128 128 128 F F F 22 440-459 83 83 83 128 128 128 F F F 23 460-479 84 84 84 129 129 129 F F F 24 480-499 87 87 87 135 135 135 F T T <--- 25 500-519 84 84 84 129 129 129 F T T <--- 26 520-539 83 88 88 128 137 137 F T T <--- 27 540-559 80 88 88 123 137 137 F F F <--- 28 560-579 85 85 85 131 131 131 F T T <--- 29 580-599 85 85 85 131 131 131 F F F Max gen: 88, Max flush: 137 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 44 Verifying data now No change after verify Loop: 44 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 81 81 81 125 125 125 F F F 1 020-039 77 77 77 117 117 117 F F F 2 040-059 85 85 85 131 131 131 F F F 3 060-079 88 88 88 137 137 137 F F F 4 080-099 86 86 86 134 134 134 F F F 5 100-119 88 88 88 136 136 136 F F F 6 120-139 84 84 84 129 129 129 F F F 7 140-159 88 88 88 137 137 137 F F F 8 160-179 84 84 84 129 129 129 F F F 9 180-199 88 88 88 136 136 136 F F F 10 200-219 85 85 85 131 131 131 F F F 11 220-239 85 85 85 131 131 131 F F F 12 240-259 82 82 82 126 126 126 F F F 13 260-279 82 82 82 126 126 126 F F F 14 280-299 86 86 86 134 134 134 F F F 15 300-319 86 86 86 134 134 134 F F F 16 320-339 87 87 87 135 135 135 F F F 17 340-359 84 84 84 129 129 129 F F F 18 360-379 84 84 84 129 129 129 F F F 19 380-399 87 87 87 135 135 135 F F F 20 400-419 82 82 82 127 127 127 F F F 21 420-439 83 83 83 128 128 128 F F F 22 440-459 83 83 83 128 128 128 F F F 23 460-479 84 84 84 129 129 129 F F F 24 480-499 89 89 89 138 138 138 F F F 25 500-519 89 89 89 138 138 138 F F F 26 520-539 89 89 89 138 138 138 F F F 27 540-559 88 88 88 137 137 137 F F F 28 560-579 89 89 89 138 138 138 F F F 29 580-599 85 85 85 131 131 131 F F F Max gen: 89, Max flush: 138 Begin loop 45 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:45.280168712Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1496,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:22:45.287449323Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1496,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:45.287499413Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1496,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:45.294186513Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1496,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:45.294453643Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1496} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:45.294488953Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1496} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:45.294497943Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1496} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:45.294509773Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1496} {"msg":"Crucible 7566d3be-13e5-412e-b0d4-f634c66aa6b7 has session id: b96e75c6-c617-4f4b-b178-179626efc307","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:45.294864044Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1496,"session_id":"b96e75c6-c617-4f4b-b178-179626efc307"} {"msg":"Upstairs opts: Upstairs UUID: 7566d3be-13e5-412e-b0d4-f634c66aa6b7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:45.294886974Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1496,"session_id":"b96e75c6-c617-4f4b-b178-179626efc307"} {"msg":"Crucible stats registered with UUID: 7566d3be-13e5-412e-b0d4-f634c66aa6b7","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:45.294899244Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1496,"session_id":"b96e75c6-c617-4f4b-b178-179626efc307"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:22:45.294908484Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1496,"session_id":"b96e75c6-c617-4f4b-b178-179626efc307"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:45.295050734Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1496,"":"io task","client":"2","":"downstairs","session_id":"b96e75c6-c617-4f4b-b178-179626efc307"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:45.295068894Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1496,"":"io task","client":"0","":"downstairs","session_id":"b96e75c6-c617-4f4b-b178-179626efc307"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:45.295079114Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1496,"":"io task","client":"1","":"downstairs","session_id":"b96e75c6-c617-4f4b-b178-179626efc307"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:45.295233294Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1496,"task":"crutest"} {"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:45.295257754Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1496} {"msg":"7566d3be-13e5-412e-b0d4-f634c66aa6b7 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:45.295267004Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1496,"session_id":"b96e75c6-c617-4f4b-b178-179626efc307"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:45.295276054Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1496,"client":"0","":"downstairs","session_id":"b96e75c6-c617-4f4b-b178-179626efc307"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:45.295285934Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1496,"client":"1","":"downstairs","session_id":"b96e75c6-c617-4f4b-b178-179626efc307"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:45.295295824Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1496,"client":"2","":"downstairs","session_id":"b96e75c6-c617-4f4b-b178-179626efc307"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:45.295305794Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1496,"":"io task","client":"2","":"downstairs","session_id":"b96e75c6-c617-4f4b-b178-179626efc307"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:45.295315934Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1496,"":"io task","client":"0","":"downstairs","session_id":"b96e75c6-c617-4f4b-b178-179626efc307"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:45.295326195Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1496,"":"io task","client":"1","":"downstairs","session_id":"b96e75c6-c617-4f4b-b178-179626efc307"} {"msg":"ds_connection connected from Ok(127.0.0.1:59941)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:45.295395555Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1496,"":"io task","client":"0","":"downstairs","session_id":"b96e75c6-c617-4f4b-b178-179626efc307"} {"msg":"ds_connection connected from Ok(127.0.0.1:51104)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:45.295421195Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1496,"":"io task","client":"1","":"downstairs","session_id":"b96e75c6-c617-4f4b-b178-179626efc307"} {"msg":"ds_connection connected from Ok(127.0.0.1:53849)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:45.295434445Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1496,"":"io task","client":"2","":"downstairs","session_id":"b96e75c6-c617-4f4b-b178-179626efc307"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:45.296111616Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1496,"client":"1","":"downstairs","session_id":"b96e75c6-c617-4f4b-b178-179626efc307"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:45.296145286Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1496,"client":"2","":"downstairs","session_id":"b96e75c6-c617-4f4b-b178-179626efc307"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:45.296175086Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1496,"client":"0","":"downstairs","session_id":"b96e75c6-c617-4f4b-b178-179626efc307"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:45.296436046Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1496,"session_id":"b96e75c6-c617-4f4b-b178-179626efc307"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:45.296453716Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1496,"session_id":"b96e75c6-c617-4f4b-b178-179626efc307"} {"msg":"[0]R flush_numbers[0..12]: [125, 117, 131, 137, 134, 136, 129, 137, 129, 136, 131, 131]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:45.296537396Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1496,"":"downstairs","":"downstairs","session_id":"b96e75c6-c617-4f4b-b178-179626efc307"} {"msg":"[0]R generation[0..12]: [81, 77, 85, 88, 86, 88, 84, 88, 84, 88, 85, 85]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:45.296548736Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1496,"":"downstairs","":"downstairs","session_id":"b96e75c6-c617-4f4b-b178-179626efc307"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:45.296560076Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1496,"":"downstairs","":"downstairs","session_id":"b96e75c6-c617-4f4b-b178-179626efc307"} {"msg":"[1]R flush_numbers[0..12]: [125, 117, 131, 137, 134, 136, 129, 137, 129, 136, 131, 131]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:45.296570846Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1496,"":"downstairs","":"downstairs","session_id":"b96e75c6-c617-4f4b-b178-179626efc307"} {"msg":"[1]R generation[0..12]: [81, 77, 85, 88, 86, 88, 84, 88, 84, 88, 85, 85]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:45.296581346Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1496,"":"downstairs","":"downstairs","session_id":"b96e75c6-c617-4f4b-b178-179626efc307"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:45.296592476Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1496,"":"downstairs","":"downstairs","session_id":"b96e75c6-c617-4f4b-b178-179626efc307"} {"msg":"[2]R flush_numbers[0..12]: [125, 117, 131, 137, 134, 136, 129, 137, 129, 136, 131, 131]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:45.296603156Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1496,"":"downstairs","":"downstairs","session_id":"b96e75c6-c617-4f4b-b178-179626efc307"} {"msg":"[2]R generation[0..12]: [81, 77, 85, 88, 86, 88, 84, 88, 84, 88, 85, 85]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:45.296615556Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1496,"":"downstairs","":"downstairs","session_id":"b96e75c6-c617-4f4b-b178-179626efc307"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:45.296626686Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1496,"":"downstairs","":"downstairs","session_id":"b96e75c6-c617-4f4b-b178-179626efc307"} {"msg":"Max found gen is 90","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:45.296636206Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1496,"":"downstairs","":"downstairs","session_id":"b96e75c6-c617-4f4b-b178-179626efc307"} {"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:45.296645987Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1496,"":"downstairs","":"downstairs","session_id":"b96e75c6-c617-4f4b-b178-179626efc307"} {"msg":"Next flush: 139","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:45.296655417Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1496,"":"downstairs","":"downstairs","session_id":"b96e75c6-c617-4f4b-b178-179626efc307"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:45.296664887Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1496,"":"downstairs","":"downstairs","session_id":"b96e75c6-c617-4f4b-b178-179626efc307"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:45.296674747Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1496,"session_id":"b96e75c6-c617-4f4b-b178-179626efc307"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:45.296683547Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1496,"client":"0","":"downstairs","session_id":"b96e75c6-c617-4f4b-b178-179626efc307"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:45.296693407Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1496,"client":"1","":"downstairs","session_id":"b96e75c6-c617-4f4b-b178-179626efc307"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:45.296703167Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1496,"client":"2","":"downstairs","session_id":"b96e75c6-c617-4f4b-b178-179626efc307"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:45.296713057Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1496,"session_id":"b96e75c6-c617-4f4b-b178-179626efc307"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:45.296722377Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1496,"session_id":"b96e75c6-c617-4f4b-b178-179626efc307"} {"msg":"7566d3be-13e5-412e-b0d4-f634c66aa6b7 is now active with session: b96e75c6-c617-4f4b-b178-179626efc307","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:45.296732737Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1496,"session_id":"b96e75c6-c617-4f4b-b178-179626efc307"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:45.296741647Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1496,"session_id":"b96e75c6-c617-4f4b-b178-179626efc307"} {"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:45.296750767Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1496} {"msg":"Activated sub_volume 7566d3be-13e5-412e-b0d4-f634c66aa6b7","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:45.296759167Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1496} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:45.296767427Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1496,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:45.297077937Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1496,"task":"crutest"} 01/30 Write block 129 len 16384 data: 4 4 5 6 02/30 Read block 164 len 32768 03/30 Write block 392 len 4096 data: 9 04/30 Write block 283 len 4096 data: 4 05/30 Write block 67 len 16384 data: 9 7 8 8 06/30 Read block 218 len 36864 07/30 Read block 156 len 36864 08/30 Read block 352 len 8192 09/30 Read block 478 len 16384 10/30 Read block 394 len 32768 11/30 Write block 12 len 36864 data: 5 5 6 6 6 6 7 9 10 12/30 Read block 158 len 8192 13/30 Read block 359 len 28672 14/30 Read block 581 len 20480 15/30 Write block 565 len 8192 data: 8 9 16/30 Write block 544 len 28672 data: 8 6 5 4 6 6 4 17/30 Write block 477 len 28672 data: 7 7 7 7 6 4 6 18/30 Write block 432 len 36864 data: 4 5 4 3 3 5 5 5 4 19/30 Write block 101 len 16384 data: 4 5 7 8 20/30 Read block 109 len 20480 21/30 Write block 270 len 12288 data: 13 12 13 22/30 Write block 216 len 28672 data: 8 8 8 7 4 4 4 23/30 Read block 234 len 28672 24/30 Read block 363 len 36864 25/30 Write block 66 len 40960 data: 9 10 8 9 9 7 7 7 5 3 26/30 Write block 330 len 8192 data: 8 9 27/30 Write block 213 len 28672 data: 6 5 6 9 9 9 8 28/30 Read block 234 len 8192 29/30 Read block 489 len 16384 30/30 Write block 261 len 36864 data: 8 9 11 10 11 9 10 11 13 ---------------------------------------------------------------- Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Read 8 Done Done Done false 1002 Acked Write 1 Done Done Done false 1003 Acked Write 1 Done Done Done false 1004 Acked Write 4 Done Done Done false 1005 Acked Read 9 Done Done Done false 1006 Acked Read 9 Done Done Done false 1007 Acked Read 2 Sent Done Done false 1008 Acked Read 4 Sent Done Done false 1009 Acked Read 8 Sent Done Done false 1010 Acked Write 9 Sent Done Done false 1011 Acked Read 2 Sent Done Done false 1012 Acked Read 7 Sent Done Done false 1013 Acked Read 5 Sent Done Done false 1014 Acked Write 2 Sent Done Done false 1015 Acked Write 7 Sent Done Done false 1016 Acked Write 7 Sent Done Done false 1017 Acked Write 9 Sent Done Done false 1018 Acked Write 4 Sent Done Done false 1019 Acked Read 5 Sent Done Done false 1020 Acked Write 3 Sent Done Done false 1021 Acked Write 7 Sent Done Done false 1022 Acked Read 7 Sent Done Done false 1023 Acked Read 9 Sent Done Done false 1024 Acked Write 10 Sent Done Done false 1025 Acked Write 2 Sent Done Done false 1026 Acked Write 7 Sent Done Done false 1027 Acked Read 2 Sent Done Done false 1028 Acked Read 4 Sent Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 23 1 1 25 Done 7 29 29 65 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:22:45.304837999Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1496} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:45.305023549Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1496,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 81 81 81 125 125 125 F T T <--- 1 020-039 77 77 77 117 117 117 F T T <--- 2 040-059 85 85 85 131 131 131 F F F 3 060-079 88 88 88 137 137 137 T T T 4 080-099 86 86 86 134 134 134 F F F 5 100-119 88 88 88 136 136 136 F T T <--- 6 120-139 84 84 84 129 129 129 T T T 7 140-159 88 88 88 137 137 137 F F F 8 160-179 84 84 84 129 129 129 F F F 9 180-199 88 88 88 136 136 136 F F F 10 200-219 85 85 85 131 131 131 F T T <--- 11 220-239 85 85 85 131 131 131 F T T <--- 12 240-259 82 82 82 126 126 126 F F F 13 260-279 82 82 82 126 126 126 F T T <--- 14 280-299 86 86 86 134 134 134 T T T 15 300-319 86 86 86 134 134 134 F F F 16 320-339 87 87 87 135 135 135 F T T <--- 17 340-359 84 84 84 129 129 129 F F F 18 360-379 84 84 84 129 129 129 F F F 19 380-399 87 87 87 135 135 135 T T T 20 400-419 82 82 82 127 127 127 F F F 21 420-439 83 83 83 128 128 128 F T T <--- 22 440-459 83 83 83 128 128 128 F T T <--- 23 460-479 84 84 84 129 129 129 F T T <--- 24 480-499 89 89 89 138 138 138 F T T <--- 25 500-519 89 89 89 138 138 138 F F F 26 520-539 89 89 89 138 138 138 F F F 27 540-559 88 88 88 137 137 137 F T T <--- 28 560-579 89 89 89 138 138 138 F T T <--- 29 580-599 85 85 85 131 131 131 F F F Max gen: 89, Max flush: 138 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 45 Verifying data now Files /tmp/test_repair/test_repair_verify.data and /tmp/test_repair/test_repair_verify.data.last differ 68a69,70 > 9, > 10, 70a73 > 9, 72,76c75,78 < 8, < 8, < 6, < 6, < 6, --- > 7, > 7, > 5, > 3, 78,79d79 < 2, < 4, diff found after verify Loop: 45 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 90 90 90 139 139 139 F F F 1 020-039 90 90 90 139 139 139 F F F 2 040-059 85 85 85 131 131 131 F F F 3 060-079 90 90 90 139 139 139 F F F 4 080-099 86 86 86 134 134 134 F F F 5 100-119 90 90 90 139 139 139 F F F 6 120-139 90 90 90 139 139 139 F F F 7 140-159 88 88 88 137 137 137 F F F 8 160-179 84 84 84 129 129 129 F F F 9 180-199 88 88 88 136 136 136 F F F 10 200-219 90 90 90 139 139 139 F F F 11 220-239 90 90 90 139 139 139 F F F 12 240-259 82 82 82 126 126 126 F F F 13 260-279 90 90 90 139 139 139 F F F 14 280-299 90 90 90 139 139 139 F F F 15 300-319 86 86 86 134 134 134 F F F 16 320-339 90 90 90 139 139 139 F F F 17 340-359 84 84 84 129 129 129 F F F 18 360-379 84 84 84 129 129 129 F F F 19 380-399 90 90 90 139 139 139 F F F 20 400-419 82 82 82 127 127 127 F F F 21 420-439 90 90 90 139 139 139 F F F 22 440-459 90 90 90 139 139 139 F F F 23 460-479 90 90 90 139 139 139 F F F 24 480-499 90 90 90 139 139 139 F F F 25 500-519 89 89 89 138 138 138 F F F 26 520-539 89 89 89 138 138 138 F F F 27 540-559 90 90 90 139 139 139 F F F 28 560-579 90 90 90 139 139 139 F F F 29 580-599 85 85 85 131 131 131 F F F Max gen: 90, Max flush: 139 Begin loop 46 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:49.6063259Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1507,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:49.614729643Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1507,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:49.615088023Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1507} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:49.615117403Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1507} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:49.615125973Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1507} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:49.615137713Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1507} {"msg":"Crucible cf626fe2-5038-4183-a950-33b9d438a05f has session id: 69132881-d14d-44a1-82f4-ae51e318215c","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:49.615501414Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1507,"session_id":"69132881-d14d-44a1-82f4-ae51e318215c"} {"msg":"Upstairs opts: Upstairs UUID: cf626fe2-5038-4183-a950-33b9d438a05f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:49.615526374Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1507,"session_id":"69132881-d14d-44a1-82f4-ae51e318215c"} {"msg":"Crucible stats registered with UUID: cf626fe2-5038-4183-a950-33b9d438a05f","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:49.615536504Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1507,"session_id":"69132881-d14d-44a1-82f4-ae51e318215c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:22:49.615545704Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1507,"session_id":"69132881-d14d-44a1-82f4-ae51e318215c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:49.615701534Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1507,"":"io task","client":"1","":"downstairs","session_id":"69132881-d14d-44a1-82f4-ae51e318215c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:49.615721224Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1507,"":"io task","client":"0","":"downstairs","session_id":"69132881-d14d-44a1-82f4-ae51e318215c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:49.615731324Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1507,"":"io task","client":"2","":"downstairs","session_id":"69132881-d14d-44a1-82f4-ae51e318215c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:49.615888864Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1507,"task":"crutest"} {"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:49.615905324Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1507} {"msg":"cf626fe2-5038-4183-a950-33b9d438a05f active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:49.615922344Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1507,"session_id":"69132881-d14d-44a1-82f4-ae51e318215c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:49.615931354Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1507,"client":"0","":"downstairs","session_id":"69132881-d14d-44a1-82f4-ae51e318215c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:49.615940984Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1507,"client":"1","":"downstairs","session_id":"69132881-d14d-44a1-82f4-ae51e318215c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:49.615950634Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1507,"client":"2","":"downstairs","session_id":"69132881-d14d-44a1-82f4-ae51e318215c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:49.615960295Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1507,"":"io task","client":"2","":"downstairs","session_id":"69132881-d14d-44a1-82f4-ae51e318215c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:49.615970375Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1507,"":"io task","client":"1","":"downstairs","session_id":"69132881-d14d-44a1-82f4-ae51e318215c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:49.615980505Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1507,"":"io task","client":"0","":"downstairs","session_id":"69132881-d14d-44a1-82f4-ae51e318215c"} {"msg":"ds_connection connected from Ok(127.0.0.1:58507)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:49.616076165Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1507,"":"io task","client":"0","":"downstairs","session_id":"69132881-d14d-44a1-82f4-ae51e318215c"} {"msg":"ds_connection connected from Ok(127.0.0.1:52988)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:49.616120925Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1507,"":"io task","client":"1","":"downstairs","session_id":"69132881-d14d-44a1-82f4-ae51e318215c"} {"msg":"ds_connection connected from Ok(127.0.0.1:64607)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:49.616177565Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1507,"":"io task","client":"2","":"downstairs","session_id":"69132881-d14d-44a1-82f4-ae51e318215c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:49.616766786Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1507,"client":"0","":"downstairs","session_id":"69132881-d14d-44a1-82f4-ae51e318215c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:49.616791586Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1507,"client":"1","":"downstairs","session_id":"69132881-d14d-44a1-82f4-ae51e318215c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:49.616813356Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1507,"client":"2","":"downstairs","session_id":"69132881-d14d-44a1-82f4-ae51e318215c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:49.617061826Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1507,"session_id":"69132881-d14d-44a1-82f4-ae51e318215c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:49.617094816Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1507,"session_id":"69132881-d14d-44a1-82f4-ae51e318215c"} {"msg":"[0]R flush_numbers[0..12]: [139, 139, 131, 139, 134, 139, 139, 137, 129, 136, 139, 139]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:49.617181306Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1507,"":"downstairs","":"downstairs","session_id":"69132881-d14d-44a1-82f4-ae51e318215c"} {"msg":"[0]R generation[0..12]: [90, 90, 85, 90, 86, 90, 90, 88, 84, 88, 90, 90]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:49.617192496Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1507,"":"downstairs","":"downstairs","session_id":"69132881-d14d-44a1-82f4-ae51e318215c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:49.617203636Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1507,"":"downstairs","":"downstairs","session_id":"69132881-d14d-44a1-82f4-ae51e318215c"} {"msg":"[1]R flush_numbers[0..12]: [139, 139, 131, 139, 134, 139, 139, 137, 129, 136, 139, 139]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:49.617214506Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1507,"":"downstairs","":"downstairs","session_id":"69132881-d14d-44a1-82f4-ae51e318215c"} {"msg":"[1]R generation[0..12]: [90, 90, 85, 90, 86, 90, 90, 88, 84, 88, 90, 90]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:49.617224876Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1507,"":"downstairs","":"downstairs","session_id":"69132881-d14d-44a1-82f4-ae51e318215c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:49.617235976Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1507,"":"downstairs","":"downstairs","session_id":"69132881-d14d-44a1-82f4-ae51e318215c"} {"msg":"[2]R flush_numbers[0..12]: [139, 139, 131, 139, 134, 139, 139, 137, 129, 136, 139, 139]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:49.617246736Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1507,"":"downstairs","":"downstairs","session_id":"69132881-d14d-44a1-82f4-ae51e318215c"} {"msg":"[2]R generation[0..12]: [90, 90, 85, 90, 86, 90, 90, 88, 84, 88, 90, 90]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:49.617257136Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1507,"":"downstairs","":"downstairs","session_id":"69132881-d14d-44a1-82f4-ae51e318215c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:49.617268166Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1507,"":"downstairs","":"downstairs","session_id":"69132881-d14d-44a1-82f4-ae51e318215c"} {"msg":"Max found gen is 91","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:49.617277726Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1507,"":"downstairs","":"downstairs","session_id":"69132881-d14d-44a1-82f4-ae51e318215c"} {"msg":"Generation requested: 92 >= found:91","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:49.617287517Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1507,"":"downstairs","":"downstairs","session_id":"69132881-d14d-44a1-82f4-ae51e318215c"} {"msg":"Next flush: 140","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:49.617298747Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1507,"":"downstairs","":"downstairs","session_id":"69132881-d14d-44a1-82f4-ae51e318215c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:49.617308127Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1507,"":"downstairs","":"downstairs","session_id":"69132881-d14d-44a1-82f4-ae51e318215c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:49.617317897Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1507,"session_id":"69132881-d14d-44a1-82f4-ae51e318215c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:49.617326797Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1507,"client":"0","":"downstairs","session_id":"69132881-d14d-44a1-82f4-ae51e318215c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:49.617336487Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1507,"client":"1","":"downstairs","session_id":"69132881-d14d-44a1-82f4-ae51e318215c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:49.617346227Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1507,"client":"2","":"downstairs","session_id":"69132881-d14d-44a1-82f4-ae51e318215c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:49.617356307Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1507,"session_id":"69132881-d14d-44a1-82f4-ae51e318215c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:49.617365417Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1507,"session_id":"69132881-d14d-44a1-82f4-ae51e318215c"} {"msg":"cf626fe2-5038-4183-a950-33b9d438a05f is now active with session: 69132881-d14d-44a1-82f4-ae51e318215c","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:49.617375367Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1507,"session_id":"69132881-d14d-44a1-82f4-ae51e318215c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:49.617384197Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1507,"session_id":"69132881-d14d-44a1-82f4-ae51e318215c"} {"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:49.617393477Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1507} {"msg":"Activated sub_volume cf626fe2-5038-4183-a950-33b9d438a05f","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:49.617401927Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1507} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:49.617410197Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1507,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:49.617735177Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1507,"task":"crutest"} 01/30 Flush 02/30 Write block 379 len 24576 data: 11 10 10 7 8 8 03/30 Write block 6 len 28672 data: 6 6 6 3 4 4 6 04/30 Read block 228 len 36864 05/30 Write block 50 len 12288 data: 7 8 9 06/30 Read block 101 len 8192 07/30 Read block 348 len 28672 08/30 Read block 564 len 32768 09/30 Read block 542 len 32768 10/30 Read block 14 len 20480 11/30 Write block 220 len 32768 data: 5 5 5 4 3 5 5 4 12/30 Write block 46 len 40960 data: 5 5 5 4 8 9 10 9 7 8 13/30 Write block 578 len 28672 data: 11 8 6 6 6 7 8 14/30 Write block 449 len 32768 data: 7 6 5 4 4 5 7 8 15/30 Write block 85 len 4096 data: 9 16/30 Read block 140 len 16384 17/30 Read block 36 len 36864 18/30 Read block 433 len 16384 19/30 Read block 451 len 16384 20/30 Read block 500 len 24576 21/30 Write block 277 len 40960 data: 12 8 6 8 6 5 5 3 3 4 22/30 Read block 36 len 12288 23/30 Read block 179 len 28672 24/30 Flush 25/30 Write block 447 len 4096 data: 6 26/30 Read block 40 len 4096 27/30 Read block 567 len 36864 28/30 Read block 416 len 16384 29/30 Write block 69 len 40960 data: 9 9 7 7 7 5 3 5 6 7 30/30 Write block 158 len 16384 data: 10 8 9 9 ---------------------------------------------------------------- Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 6 Done Done Done false 1002 Acked Write 7 Done Done Done false 1003 Acked Read 9 Done Done Done false 1004 Acked Write 3 Done Done Done false 1005 Acked Read 2 Done Done Done false 1006 Acked Read 7 Done Done Done false 1007 Acked Read 8 Done Done Done false 1008 Acked Read 8 Done Done Done false 1009 Acked Read 5 Done Done Sent false 1010 Acked Write 8 Done Done Sent false 1011 Acked Write 10 Done Done Sent false 1012 Acked Write 7 Done Done Sent false 1013 Acked Write 8 Done Done Sent false 1014 Acked Write 1 Done Done Sent false 1015 Acked Read 4 Done Done Sent false 1016 Acked Read 9 Done Done Sent false 1017 Acked Read 4 Done Done Sent false 1018 Acked Read 4 Done Done Sent false 1019 Acked Read 6 Done Done Sent false 1020 Acked Write 10 Done Done Sent false 1021 Acked Read 3 Done Done Sent false 1022 Acked Read 7 Done Done Sent false 1023 Acked Flush 0 Done Done Sent false 1024 Acked Write 1 Done Done Sent false 1025 Acked Read 1 Done Done Sent false 1026 Acked Read 9 Done Done Sent false 1027 Acked Read 4 Done Done Sent false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 21 25 Done 27 27 8 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1000)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:22:49.626749161Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1507} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:49.626917721Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1507,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 90 141 141 139 F F T <--- 1 020-039 90 90 90 139 139 139 F F F 2 040-059 92 92 85 141 141 131 F F T <--- 3 060-079 90 90 90 139 139 139 T T F <--- 4 080-099 92 92 86 141 141 134 F F F <--- 5 100-119 90 90 90 139 139 139 F F F 6 120-139 90 90 90 139 139 139 F F F 7 140-159 88 88 88 137 137 137 T T F <--- 8 160-179 84 84 84 129 129 129 T T F <--- 9 180-199 88 88 88 136 136 136 F F F 10 200-219 90 90 90 139 139 139 F F F 11 220-239 92 92 90 141 141 139 F F F <--- 12 240-259 82 82 82 126 126 126 F F F 13 260-279 92 92 90 141 141 139 F F F <--- 14 280-299 92 92 90 141 141 139 F F F <--- 15 300-319 86 86 86 134 134 134 F F F 16 320-339 90 90 90 139 139 139 F F F 17 340-359 84 84 84 129 129 129 F F F 18 360-379 92 92 84 141 141 129 F F T <--- 19 380-399 92 92 90 141 141 139 F F T <--- 20 400-419 82 82 82 127 127 127 F F F 21 420-439 90 90 90 139 139 139 F F F 22 440-459 92 92 90 141 141 139 T T F <--- 23 460-479 90 90 90 139 139 139 F F F 24 480-499 90 90 90 139 139 139 F F F 25 500-519 89 89 89 138 138 138 F F F 26 520-539 89 89 89 138 138 138 F F F 27 540-559 90 90 90 139 139 139 F F F 28 560-579 92 92 90 141 141 139 F F F <--- 29 580-599 92 92 85 141 141 131 F F F <--- Max gen: 92, Max flush: 141 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 46 Verifying data now No change after verify Loop: 46 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 141 141 141 F F F 1 020-039 90 90 90 139 139 139 F F F 2 040-059 92 92 92 141 141 141 F F F 3 060-079 93 93 93 142 142 142 F F F 4 080-099 92 92 92 141 141 141 F F F 5 100-119 90 90 90 139 139 139 F F F 6 120-139 90 90 90 139 139 139 F F F 7 140-159 93 93 93 142 142 142 F F F 8 160-179 93 93 93 142 142 142 F F F 9 180-199 88 88 88 136 136 136 F F F 10 200-219 90 90 90 139 139 139 F F F 11 220-239 92 92 92 141 141 141 F F F 12 240-259 82 82 82 126 126 126 F F F 13 260-279 92 92 92 141 141 141 F F F 14 280-299 92 92 92 141 141 141 F F F 15 300-319 86 86 86 134 134 134 F F F 16 320-339 90 90 90 139 139 139 F F F 17 340-359 84 84 84 129 129 129 F F F 18 360-379 92 92 92 141 141 141 F F F 19 380-399 92 92 92 141 141 141 F F F 20 400-419 82 82 82 127 127 127 F F F 21 420-439 90 90 90 139 139 139 F F F 22 440-459 93 93 93 142 142 142 F F F 23 460-479 90 90 90 139 139 139 F F F 24 480-499 90 90 90 139 139 139 F F F 25 500-519 89 89 89 138 138 138 F F F 26 520-539 89 89 89 138 138 138 F F F 27 540-559 90 90 90 139 139 139 F F F 28 560-579 92 92 92 141 141 141 F F F 29 580-599 92 92 92 141 141 141 F F F Max gen: 93, Max flush: 142 Begin loop 47 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:53.881928241Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1515,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:53.890027644Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1515,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:53.890324654Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1515} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:53.890351474Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1515} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:53.890363454Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1515} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:53.890375464Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1515} {"msg":"Crucible 6db502cd-1066-4a78-8e94-b98338e874a8 has session id: cb9a7486-9ce0-4a75-8de6-6149b885e08d","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:53.890803435Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1515,"session_id":"cb9a7486-9ce0-4a75-8de6-6149b885e08d"} {"msg":"Upstairs opts: Upstairs UUID: 6db502cd-1066-4a78-8e94-b98338e874a8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:53.890832035Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1515,"session_id":"cb9a7486-9ce0-4a75-8de6-6149b885e08d"} {"msg":"Crucible stats registered with UUID: 6db502cd-1066-4a78-8e94-b98338e874a8","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:53.890843225Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1515,"session_id":"cb9a7486-9ce0-4a75-8de6-6149b885e08d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:22:53.890852555Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1515,"session_id":"cb9a7486-9ce0-4a75-8de6-6149b885e08d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:53.890971325Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1515,"":"io task","client":"2","":"downstairs","session_id":"cb9a7486-9ce0-4a75-8de6-6149b885e08d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:53.890988915Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1515,"":"io task","client":"0","":"downstairs","session_id":"cb9a7486-9ce0-4a75-8de6-6149b885e08d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:53.891006785Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1515,"":"io task","client":"1","":"downstairs","session_id":"cb9a7486-9ce0-4a75-8de6-6149b885e08d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:53.891135445Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1515,"task":"crutest"} {"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:53.891150405Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1515} {"msg":"6db502cd-1066-4a78-8e94-b98338e874a8 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:53.891173655Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1515,"session_id":"cb9a7486-9ce0-4a75-8de6-6149b885e08d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:53.891183245Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1515,"client":"0","":"downstairs","session_id":"cb9a7486-9ce0-4a75-8de6-6149b885e08d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:53.891194775Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1515,"client":"1","":"downstairs","session_id":"cb9a7486-9ce0-4a75-8de6-6149b885e08d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:53.891204615Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1515,"client":"2","":"downstairs","session_id":"cb9a7486-9ce0-4a75-8de6-6149b885e08d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:53.891214306Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1515,"":"io task","client":"2","":"downstairs","session_id":"cb9a7486-9ce0-4a75-8de6-6149b885e08d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:53.891224336Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1515,"":"io task","client":"0","":"downstairs","session_id":"cb9a7486-9ce0-4a75-8de6-6149b885e08d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:53.891234436Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1515,"":"io task","client":"1","":"downstairs","session_id":"cb9a7486-9ce0-4a75-8de6-6149b885e08d"} {"msg":"ds_connection connected from Ok(127.0.0.1:41893)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:53.891376546Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1515,"":"io task","client":"2","":"downstairs","session_id":"cb9a7486-9ce0-4a75-8de6-6149b885e08d"} {"msg":"ds_connection connected from Ok(127.0.0.1:44535)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:53.891464916Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1515,"":"io task","client":"1","":"downstairs","session_id":"cb9a7486-9ce0-4a75-8de6-6149b885e08d"} {"msg":"ds_connection connected from Ok(127.0.0.1:34867)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:53.891476466Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1515,"":"io task","client":"0","":"downstairs","session_id":"cb9a7486-9ce0-4a75-8de6-6149b885e08d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:53.892048537Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1515,"client":"2","":"downstairs","session_id":"cb9a7486-9ce0-4a75-8de6-6149b885e08d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:53.892075007Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1515,"client":"0","":"downstairs","session_id":"cb9a7486-9ce0-4a75-8de6-6149b885e08d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:53.892087057Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1515,"client":"1","":"downstairs","session_id":"cb9a7486-9ce0-4a75-8de6-6149b885e08d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:53.892374107Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1515,"session_id":"cb9a7486-9ce0-4a75-8de6-6149b885e08d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:53.892410327Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1515,"session_id":"cb9a7486-9ce0-4a75-8de6-6149b885e08d"} {"msg":"[0]R flush_numbers[0..12]: [141, 139, 141, 142, 141, 139, 139, 142, 142, 136, 139, 141]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:53.892482307Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1515,"":"downstairs","":"downstairs","session_id":"cb9a7486-9ce0-4a75-8de6-6149b885e08d"} {"msg":"[0]R generation[0..12]: [92, 90, 92, 93, 92, 90, 90, 93, 93, 88, 90, 92]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:53.892493617Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1515,"":"downstairs","":"downstairs","session_id":"cb9a7486-9ce0-4a75-8de6-6149b885e08d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:53.892507147Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1515,"":"downstairs","":"downstairs","session_id":"cb9a7486-9ce0-4a75-8de6-6149b885e08d"} {"msg":"[1]R flush_numbers[0..12]: [141, 139, 141, 142, 141, 139, 139, 142, 142, 136, 139, 141]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:53.892517987Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1515,"":"downstairs","":"downstairs","session_id":"cb9a7486-9ce0-4a75-8de6-6149b885e08d"} {"msg":"[1]R generation[0..12]: [92, 90, 92, 93, 92, 90, 90, 93, 93, 88, 90, 92]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:53.892528447Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1515,"":"downstairs","":"downstairs","session_id":"cb9a7486-9ce0-4a75-8de6-6149b885e08d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:53.892539548Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1515,"":"downstairs","":"downstairs","session_id":"cb9a7486-9ce0-4a75-8de6-6149b885e08d"} {"msg":"[2]R flush_numbers[0..12]: [141, 139, 141, 142, 141, 139, 139, 142, 142, 136, 139, 141]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:53.892550178Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1515,"":"downstairs","":"downstairs","session_id":"cb9a7486-9ce0-4a75-8de6-6149b885e08d"} {"msg":"[2]R generation[0..12]: [92, 90, 92, 93, 92, 90, 90, 93, 93, 88, 90, 92]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:53.892560658Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1515,"":"downstairs","":"downstairs","session_id":"cb9a7486-9ce0-4a75-8de6-6149b885e08d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:53.892571798Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1515,"":"downstairs","":"downstairs","session_id":"cb9a7486-9ce0-4a75-8de6-6149b885e08d"} {"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:53.892581278Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1515,"":"downstairs","":"downstairs","session_id":"cb9a7486-9ce0-4a75-8de6-6149b885e08d"} {"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:53.892591098Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1515,"":"downstairs","":"downstairs","session_id":"cb9a7486-9ce0-4a75-8de6-6149b885e08d"} {"msg":"Next flush: 143","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:53.892600638Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1515,"":"downstairs","":"downstairs","session_id":"cb9a7486-9ce0-4a75-8de6-6149b885e08d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:53.892609978Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1515,"":"downstairs","":"downstairs","session_id":"cb9a7486-9ce0-4a75-8de6-6149b885e08d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:53.892620058Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1515,"session_id":"cb9a7486-9ce0-4a75-8de6-6149b885e08d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:53.892629008Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1515,"client":"0","":"downstairs","session_id":"cb9a7486-9ce0-4a75-8de6-6149b885e08d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:53.892638778Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1515,"client":"1","":"downstairs","session_id":"cb9a7486-9ce0-4a75-8de6-6149b885e08d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:53.892648508Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1515,"client":"2","":"downstairs","session_id":"cb9a7486-9ce0-4a75-8de6-6149b885e08d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:53.892659828Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1515,"session_id":"cb9a7486-9ce0-4a75-8de6-6149b885e08d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:53.892669048Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1515,"session_id":"cb9a7486-9ce0-4a75-8de6-6149b885e08d"} {"msg":"6db502cd-1066-4a78-8e94-b98338e874a8 is now active with session: cb9a7486-9ce0-4a75-8de6-6149b885e08d","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:53.892679098Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1515,"session_id":"cb9a7486-9ce0-4a75-8de6-6149b885e08d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:53.892687868Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1515,"session_id":"cb9a7486-9ce0-4a75-8de6-6149b885e08d"} {"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:53.892697128Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1515} {"msg":"Activated sub_volume 6db502cd-1066-4a78-8e94-b98338e874a8","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:53.892705308Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1515} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:53.892713498Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1515,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:53.893052748Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1515,"task":"crutest"} 01/30 Write block 113 len 40960 data: 8 8 8 8 6 7 6 6 7 7 02/30 Read block 485 len 28672 03/30 Write block 399 len 28672 data: 7 6 4 4 4 6 6 04/30 Read block 349 len 16384 05/30 Flush 06/30 Write block 240 len 8192 data: 11 11 07/30 Read block 44 len 8192 08/30 Flush 09/30 Write block 264 len 24576 data: 11 12 10 11 12 14 10/30 Write block 402 len 4096 data: 5 11/30 Flush 12/30 Write block 590 len 12288 data: 5 4 5 13/30 Write block 360 len 20480 data: 14 10 10 10 12 14/30 Flush 15/30 Write block 61 len 12288 data: 11 11 10 16/30 Read block 352 len 20480 17/30 Read block 301 len 12288 18/30 Flush 19/30 Write block 352 len 16384 data: 6 8 10 10 20/30 Write block 17 len 32768 data: 7 8 10 11 11 9 9 8 21/30 Read block 331 len 4096 22/30 Read block 308 len 28672 23/30 Flush 24/30 Write block 109 len 16384 data: 9 10 9 7 25/30 Read block 93 len 32768 26/30 Flush 27/30 Write block 512 len 4096 data: 8 28/30 Read block 576 len 36864 29/30 Write block 460 len 4096 data: 6 30/30 Write block 172 len 8192 data: 7 7 ---------------------------------------------------------------- Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:22 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1008 Acked Write 6 Done Done Done false 1009 Acked Write 1 Done Done Done false 1010 Acked Flush 0 Done Sent Done false 1011 Acked Write 3 Done Sent Done false 1012 Acked Write 5 Done Sent Done false 1013 Acked Flush 0 Done Sent Done false 1014 Acked Write 3 Done Sent Done false 1015 Acked Read 5 Done Sent Done false 1016 Acked Read 3 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Write 4 Done Sent Done false 1019 Acked Write 8 Done Sent Done false 1020 Acked Read 1 Done Sent Done false 1021 Acked Read 7 Done Sent Done false 1022 Acked Flush 0 Done Sent Done false 1023 Acked Write 4 Done Sent Done false 1024 Acked Read 8 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 1 Done Sent Done false 1027 Acked Read 9 Done Sent Done false 1028 Acked Write 1 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 20 2 24 Done 20 2 20 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1007)) Some(JobId(1025)) Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:22:53.901530921Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1515} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:53.901721011Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1515,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 94 92 94 148 141 148 F F F <--- 1 020-039 94 90 94 148 139 148 F F F <--- 2 040-059 92 92 92 141 141 141 F F F 3 060-079 94 93 94 147 142 147 F F F <--- 4 080-099 92 92 92 141 141 141 F F F 5 100-119 94 94 94 149 143 149 F F F <--- 6 120-139 94 94 94 143 143 143 F F F 7 140-159 93 93 93 142 142 142 F F F 8 160-179 93 93 93 142 142 142 T F T <--- 9 180-199 88 88 88 136 136 136 F F F 10 200-219 90 90 90 139 139 139 F F F 11 220-239 92 92 92 141 141 141 F F F 12 240-259 94 94 94 144 144 144 F F F 13 260-279 94 92 94 145 141 145 F T F <--- 14 280-299 92 92 92 141 141 141 F F F 15 300-319 86 86 86 134 134 134 F F F 16 320-339 90 90 90 139 139 139 F F F 17 340-359 94 84 94 148 129 148 F F F <--- 18 360-379 94 92 94 146 141 146 F F F <--- 19 380-399 94 94 94 143 143 143 F F F 20 400-419 94 94 94 145 143 145 F T F <--- 21 420-439 90 90 90 139 139 139 F F F 22 440-459 93 93 93 142 142 142 F F F 23 460-479 90 90 90 139 139 139 T F T <--- 24 480-499 90 90 90 139 139 139 F F F 25 500-519 89 89 89 138 138 138 T F T <--- 26 520-539 89 89 89 138 138 138 F F F 27 540-559 90 90 90 139 139 139 F F F 28 560-579 92 92 92 141 141 141 F F F 29 580-599 94 92 94 146 141 146 F F F <--- Max gen: 94, Max flush: 149 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 47 Verifying data now No change after verify Loop: 47 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 94 94 94 148 148 148 F F F 1 020-039 94 94 94 148 148 148 F F F 2 040-059 92 92 92 141 141 141 F F F 3 060-079 94 94 94 147 147 147 F F F 4 080-099 92 92 92 141 141 141 F F F 5 100-119 94 94 94 149 149 149 F F F 6 120-139 94 94 94 143 143 143 F F F 7 140-159 93 93 93 142 142 142 F F F 8 160-179 95 95 95 150 150 150 F F F 9 180-199 88 88 88 136 136 136 F F F 10 200-219 90 90 90 139 139 139 F F F 11 220-239 92 92 92 141 141 141 F F F 12 240-259 94 94 94 144 144 144 F F F 13 260-279 94 94 94 145 145 145 F F F 14 280-299 92 92 92 141 141 141 F F F 15 300-319 86 86 86 134 134 134 F F F 16 320-339 90 90 90 139 139 139 F F F 17 340-359 94 94 94 148 148 148 F F F 18 360-379 94 94 94 146 146 146 F F F 19 380-399 94 94 94 143 143 143 F F F 20 400-419 94 94 94 145 145 145 F F F 21 420-439 90 90 90 139 139 139 F F F 22 440-459 93 93 93 142 142 142 F F F 23 460-479 95 95 95 150 150 150 F F F 24 480-499 90 90 90 139 139 139 F F F 25 500-519 95 95 95 150 150 150 F F F 26 520-539 89 89 89 138 138 138 F F F 27 540-559 90 90 90 139 139 139 F F F 28 560-579 92 92 92 141 141 141 F F F 29 580-599 94 94 94 146 146 146 F F F Max gen: 95, Max flush: 150 Begin loop 48 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:58.147957049Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1523,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:58.156031711Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1523,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:58.156319242Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1523} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:58.156350662Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1523} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:58.156359632Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1523} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:58.156371472Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1523} {"msg":"Crucible 469c84ac-bcf4-4d58-8ce4-f53d56f5652e has session id: 0e46e045-f3a7-4a0f-a2ab-9862b6e60f06","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:58.156702622Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1523,"session_id":"0e46e045-f3a7-4a0f-a2ab-9862b6e60f06"} {"msg":"Upstairs opts: Upstairs UUID: 469c84ac-bcf4-4d58-8ce4-f53d56f5652e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:58.156740082Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1523,"session_id":"0e46e045-f3a7-4a0f-a2ab-9862b6e60f06"} {"msg":"Crucible stats registered with UUID: 469c84ac-bcf4-4d58-8ce4-f53d56f5652e","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:58.156751202Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1523,"session_id":"0e46e045-f3a7-4a0f-a2ab-9862b6e60f06"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:22:58.156762482Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1523,"session_id":"0e46e045-f3a7-4a0f-a2ab-9862b6e60f06"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:58.156908503Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1523,"":"io task","client":"0","":"downstairs","session_id":"0e46e045-f3a7-4a0f-a2ab-9862b6e60f06"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:58.156940573Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1523,"":"io task","client":"1","":"downstairs","session_id":"0e46e045-f3a7-4a0f-a2ab-9862b6e60f06"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:58.156951613Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1523,"":"io task","client":"2","":"downstairs","session_id":"0e46e045-f3a7-4a0f-a2ab-9862b6e60f06"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:58.157104693Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1523,"task":"crutest"} {"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:58.157120933Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1523} {"msg":"469c84ac-bcf4-4d58-8ce4-f53d56f5652e active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:58.157137193Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1523,"session_id":"0e46e045-f3a7-4a0f-a2ab-9862b6e60f06"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:58.157146213Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1523,"client":"0","":"downstairs","session_id":"0e46e045-f3a7-4a0f-a2ab-9862b6e60f06"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:58.157156013Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1523,"client":"1","":"downstairs","session_id":"0e46e045-f3a7-4a0f-a2ab-9862b6e60f06"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:58.157165633Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1523,"client":"2","":"downstairs","session_id":"0e46e045-f3a7-4a0f-a2ab-9862b6e60f06"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:58.157175273Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1523,"":"io task","client":"2","":"downstairs","session_id":"0e46e045-f3a7-4a0f-a2ab-9862b6e60f06"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:58.157185283Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1523,"":"io task","client":"0","":"downstairs","session_id":"0e46e045-f3a7-4a0f-a2ab-9862b6e60f06"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:58.157195363Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1523,"":"io task","client":"1","":"downstairs","session_id":"0e46e045-f3a7-4a0f-a2ab-9862b6e60f06"} {"msg":"ds_connection connected from Ok(127.0.0.1:44716)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:58.157263303Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1523,"":"io task","client":"0","":"downstairs","session_id":"0e46e045-f3a7-4a0f-a2ab-9862b6e60f06"} {"msg":"ds_connection connected from Ok(127.0.0.1:63368)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:58.157288823Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1523,"":"io task","client":"2","":"downstairs","session_id":"0e46e045-f3a7-4a0f-a2ab-9862b6e60f06"} {"msg":"ds_connection connected from Ok(127.0.0.1:64237)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:58.157405503Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1523,"":"io task","client":"1","":"downstairs","session_id":"0e46e045-f3a7-4a0f-a2ab-9862b6e60f06"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:58.157963204Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1523,"client":"2","":"downstairs","session_id":"0e46e045-f3a7-4a0f-a2ab-9862b6e60f06"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:58.157987004Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1523,"client":"1","":"downstairs","session_id":"0e46e045-f3a7-4a0f-a2ab-9862b6e60f06"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:58.158010524Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1523,"client":"0","":"downstairs","session_id":"0e46e045-f3a7-4a0f-a2ab-9862b6e60f06"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:58.158273485Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1523,"session_id":"0e46e045-f3a7-4a0f-a2ab-9862b6e60f06"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:58.158302025Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1523,"session_id":"0e46e045-f3a7-4a0f-a2ab-9862b6e60f06"} {"msg":"[0]R flush_numbers[0..12]: [148, 148, 141, 147, 141, 149, 143, 142, 150, 136, 139, 141]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:58.158347855Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1523,"":"downstairs","":"downstairs","session_id":"0e46e045-f3a7-4a0f-a2ab-9862b6e60f06"} {"msg":"[0]R generation[0..12]: [94, 94, 92, 94, 92, 94, 94, 93, 95, 88, 90, 92]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:58.158359625Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1523,"":"downstairs","":"downstairs","session_id":"0e46e045-f3a7-4a0f-a2ab-9862b6e60f06"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:58.158370875Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1523,"":"downstairs","":"downstairs","session_id":"0e46e045-f3a7-4a0f-a2ab-9862b6e60f06"} {"msg":"[1]R flush_numbers[0..12]: [148, 148, 141, 147, 141, 149, 143, 142, 150, 136, 139, 141]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:58.158381675Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1523,"":"downstairs","":"downstairs","session_id":"0e46e045-f3a7-4a0f-a2ab-9862b6e60f06"} {"msg":"[1]R generation[0..12]: [94, 94, 92, 94, 92, 94, 94, 93, 95, 88, 90, 92]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:58.158392215Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1523,"":"downstairs","":"downstairs","session_id":"0e46e045-f3a7-4a0f-a2ab-9862b6e60f06"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:58.158403385Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1523,"":"downstairs","":"downstairs","session_id":"0e46e045-f3a7-4a0f-a2ab-9862b6e60f06"} {"msg":"[2]R flush_numbers[0..12]: [148, 148, 141, 147, 141, 149, 143, 142, 150, 136, 139, 141]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:58.158414205Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1523,"":"downstairs","":"downstairs","session_id":"0e46e045-f3a7-4a0f-a2ab-9862b6e60f06"} {"msg":"[2]R generation[0..12]: [94, 94, 92, 94, 92, 94, 94, 93, 95, 88, 90, 92]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:58.158424645Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1523,"":"downstairs","":"downstairs","session_id":"0e46e045-f3a7-4a0f-a2ab-9862b6e60f06"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:58.158435725Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1523,"":"downstairs","":"downstairs","session_id":"0e46e045-f3a7-4a0f-a2ab-9862b6e60f06"} {"msg":"Max found gen is 96","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:58.158447675Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1523,"":"downstairs","":"downstairs","session_id":"0e46e045-f3a7-4a0f-a2ab-9862b6e60f06"} {"msg":"Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:58.158457655Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1523,"":"downstairs","":"downstairs","session_id":"0e46e045-f3a7-4a0f-a2ab-9862b6e60f06"} {"msg":"Next flush: 151","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:58.158467165Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1523,"":"downstairs","":"downstairs","session_id":"0e46e045-f3a7-4a0f-a2ab-9862b6e60f06"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:58.158476765Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1523,"":"downstairs","":"downstairs","session_id":"0e46e045-f3a7-4a0f-a2ab-9862b6e60f06"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:58.158486785Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1523,"session_id":"0e46e045-f3a7-4a0f-a2ab-9862b6e60f06"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:58.158495745Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1523,"client":"0","":"downstairs","session_id":"0e46e045-f3a7-4a0f-a2ab-9862b6e60f06"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:58.158505785Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1523,"client":"1","":"downstairs","session_id":"0e46e045-f3a7-4a0f-a2ab-9862b6e60f06"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:58.158515565Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1523,"client":"2","":"downstairs","session_id":"0e46e045-f3a7-4a0f-a2ab-9862b6e60f06"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:58.158525595Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1523,"session_id":"0e46e045-f3a7-4a0f-a2ab-9862b6e60f06"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:58.158534875Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1523,"session_id":"0e46e045-f3a7-4a0f-a2ab-9862b6e60f06"} {"msg":"469c84ac-bcf4-4d58-8ce4-f53d56f5652e is now active with session: 0e46e045-f3a7-4a0f-a2ab-9862b6e60f06","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:58.158545115Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1523,"session_id":"0e46e045-f3a7-4a0f-a2ab-9862b6e60f06"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:58.158554035Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1523,"session_id":"0e46e045-f3a7-4a0f-a2ab-9862b6e60f06"} {"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:58.158563195Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1523} {"msg":"Activated sub_volume 469c84ac-bcf4-4d58-8ce4-f53d56f5652e","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:58.158571395Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1523} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:58.158579505Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1523,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:58.158876626Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1523,"task":"crutest"} 01/30 Write block 278 len 40960 data: 9 7 9 7 6 6 4 4 5 5 02/30 Write block 375 len 32768 data: 11 12 10 10 12 11 11 8 03/30 Write block 83 len 36864 data: 7 7 10 10 9 8 9 8 7 04/30 Read block 20 len 28672 05/30 Read block 484 len 20480 06/30 Write block 437 len 36864 data: 6 6 6 5 4 5 5 6 5 07/30 Read block 480 len 40960 08/30 Write block 263 len 8192 data: 12 12 09/30 Read block 295 len 16384 10/30 Write block 458 len 24576 data: 11 9 7 6 5 8 11/30 Read block 213 len 40960 12/30 Read block 105 len 16384 13/30 Read block 414 len 12288 14/30 Read block 356 len 4096 15/30 Read block 142 len 8192 16/30 Write block 280 len 12288 data: 10 8 7 17/30 Read block 504 len 28672 18/30 Read block 389 len 16384 19/30 Read block 376 len 16384 20/30 Read block 363 len 28672 21/30 Read block 386 len 12288 22/30 Write block 142 len 4096 data: 6 23/30 Read block 14 len 8192 24/30 Read block 466 len 20480 25/30 Read block 579 len 36864 26/30 Write block 145 len 12288 data: 6 6 7 27/30 Write block 223 len 12288 data: 5 4 6 28/30 Read block 36 len 20480 29/30 Read block 274 len 16384 30/30 Read block 247 len 32768 ---------------------------------------------------------------- Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Sent Done false 1001 Acked Write 8 Done Sent Done false 1002 Acked Write 9 Done Sent Done false 1003 Acked Read 7 Done Sent Done false 1004 Acked Read 5 Done Sent Done false 1005 Acked Write 9 Done Sent Done false 1006 Acked Read 10 Done Sent Done false 1007 Acked Write 2 Done Sent Done false 1008 Acked Read 4 Done Sent Done false 1009 Acked Write 6 Done Sent Done false 1010 Acked Read 10 Done Sent Done false 1011 Acked Read 4 Done Sent Done false 1012 Acked Read 3 Done Sent Done false 1013 Acked Read 1 Done Sent Done false 1014 Acked Read 2 Done Sent Done false 1015 Acked Write 3 Done Sent Done false 1016 Acked Read 7 Done Sent Done false 1017 Acked Read 4 Done Sent Done false 1018 Acked Read 4 Done Sent Done false 1019 Acked Read 7 Done Sent Done false 1020 Acked Read 3 Done Sent Done false 1021 Acked Write 1 Done Sent Done false 1022 Acked Read 2 Done Sent Done false 1023 Acked Read 5 Done Sent Done false 1024 Acked Read 9 Done Sent Done false 1025 Acked Write 3 Done Sent Done false 1026 Acked Write 3 Done Sent Done false 1027 Acked Read 5 Done Sent Done false 1028 Acked Read 4 Done Sent Done false 1029 Acked Read 8 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 30 0 30 Done 30 0 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:22:58.167263148Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1523} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:22:58.167450818Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1523,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 94 94 94 148 148 148 F F F 1 020-039 94 94 94 148 148 148 F F F 2 040-059 92 92 92 141 141 141 F F F 3 060-079 94 94 94 147 147 147 F F F 4 080-099 92 92 92 141 141 141 T F T <--- 5 100-119 94 94 94 149 149 149 F F F 6 120-139 94 94 94 143 143 143 F F F 7 140-159 93 93 93 142 142 142 T F T <--- 8 160-179 95 95 95 150 150 150 F F F 9 180-199 88 88 88 136 136 136 F F F 10 200-219 90 90 90 139 139 139 F F F 11 220-239 92 92 92 141 141 141 T F T <--- 12 240-259 94 94 94 144 144 144 F F F 13 260-279 94 94 94 145 145 145 T F T <--- 14 280-299 92 92 92 141 141 141 T F T <--- 15 300-319 86 86 86 134 134 134 F F F 16 320-339 90 90 90 139 139 139 F F F 17 340-359 94 94 94 148 148 148 F F F 18 360-379 94 94 94 146 146 146 T F T <--- 19 380-399 94 94 94 143 143 143 T F T <--- 20 400-419 94 94 94 145 145 145 F F F 21 420-439 90 90 90 139 139 139 T F T <--- 22 440-459 93 93 93 142 142 142 T F T <--- 23 460-479 95 95 95 150 150 150 T F T <--- 24 480-499 90 90 90 139 139 139 F F F 25 500-519 95 95 95 150 150 150 F F F 26 520-539 89 89 89 138 138 138 F F F 27 540-559 90 90 90 139 139 139 F F F 28 560-579 92 92 92 141 141 141 F F F 29 580-599 94 94 94 146 146 146 F F F Max gen: 95, Max flush: 150 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 48 Verifying data now No change after verify Loop: 48 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 94 94 94 148 148 148 F F F 1 020-039 94 94 94 148 148 148 F F F 2 040-059 92 92 92 141 141 141 F F F 3 060-079 94 94 94 147 147 147 F F F 4 080-099 96 96 96 151 151 151 F F F 5 100-119 94 94 94 149 149 149 F F F 6 120-139 94 94 94 143 143 143 F F F 7 140-159 96 96 96 151 151 151 F F F 8 160-179 95 95 95 150 150 150 F F F 9 180-199 88 88 88 136 136 136 F F F 10 200-219 90 90 90 139 139 139 F F F 11 220-239 96 96 96 151 151 151 F F F 12 240-259 94 94 94 144 144 144 F F F 13 260-279 96 96 96 151 151 151 F F F 14 280-299 96 96 96 151 151 151 F F F 15 300-319 86 86 86 134 134 134 F F F 16 320-339 90 90 90 139 139 139 F F F 17 340-359 94 94 94 148 148 148 F F F 18 360-379 96 96 96 151 151 151 F F F 19 380-399 96 96 96 151 151 151 F F F 20 400-419 94 94 94 145 145 145 F F F 21 420-439 96 96 96 151 151 151 F F F 22 440-459 96 96 96 151 151 151 F F F 23 460-479 96 96 96 151 151 151 F F F 24 480-499 90 90 90 139 139 139 F F F 25 500-519 95 95 95 150 150 150 F F F 26 520-539 89 89 89 138 138 138 F F F 27 540-559 90 90 90 139 139 139 F F F 28 560-579 92 92 92 141 141 141 F F F 29 580-599 94 94 94 146 146 146 F F F Max gen: 96, Max flush: 151 Begin loop 49 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:02.412111184Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1531,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:02.420242646Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1531,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:02.420534327Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1531} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:02.420560537Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1531} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:02.420569337Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1531} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:02.420581337Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1531} {"msg":"Crucible e73d92d0-0df2-49de-94bb-b7af1c6aa4c0 has session id: 2a9e833e-1016-43c8-92cb-a6e8aa63f69d","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:02.420959557Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1531,"session_id":"2a9e833e-1016-43c8-92cb-a6e8aa63f69d"} {"msg":"Upstairs opts: Upstairs UUID: e73d92d0-0df2-49de-94bb-b7af1c6aa4c0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:02.420980867Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1531,"session_id":"2a9e833e-1016-43c8-92cb-a6e8aa63f69d"} {"msg":"Crucible stats registered with UUID: e73d92d0-0df2-49de-94bb-b7af1c6aa4c0","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:02.420991247Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1531,"session_id":"2a9e833e-1016-43c8-92cb-a6e8aa63f69d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:23:02.421000447Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1531,"session_id":"2a9e833e-1016-43c8-92cb-a6e8aa63f69d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:02.421172517Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1531,"":"io task","client":"0","":"downstairs","session_id":"2a9e833e-1016-43c8-92cb-a6e8aa63f69d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:02.421216578Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1531,"":"io task","client":"1","":"downstairs","session_id":"2a9e833e-1016-43c8-92cb-a6e8aa63f69d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:02.421227078Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1531,"":"io task","client":"2","":"downstairs","session_id":"2a9e833e-1016-43c8-92cb-a6e8aa63f69d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:02.421341928Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1531,"task":"crutest"} {"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:02.421358448Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1531} {"msg":"e73d92d0-0df2-49de-94bb-b7af1c6aa4c0 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:02.421378148Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1531,"session_id":"2a9e833e-1016-43c8-92cb-a6e8aa63f69d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:02.421387138Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1531,"client":"0","":"downstairs","session_id":"2a9e833e-1016-43c8-92cb-a6e8aa63f69d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:02.421396938Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1531,"client":"1","":"downstairs","session_id":"2a9e833e-1016-43c8-92cb-a6e8aa63f69d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:02.421406798Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1531,"client":"2","":"downstairs","session_id":"2a9e833e-1016-43c8-92cb-a6e8aa63f69d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:02.421416588Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1531,"":"io task","client":"2","":"downstairs","session_id":"2a9e833e-1016-43c8-92cb-a6e8aa63f69d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:02.421426638Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1531,"":"io task","client":"0","":"downstairs","session_id":"2a9e833e-1016-43c8-92cb-a6e8aa63f69d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:02.421436678Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1531,"":"io task","client":"1","":"downstairs","session_id":"2a9e833e-1016-43c8-92cb-a6e8aa63f69d"} {"msg":"ds_connection connected from Ok(127.0.0.1:42155)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:02.421508708Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1531,"":"io task","client":"1","":"downstairs","session_id":"2a9e833e-1016-43c8-92cb-a6e8aa63f69d"} {"msg":"ds_connection connected from Ok(127.0.0.1:39330)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:02.421532718Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1531,"":"io task","client":"0","":"downstairs","session_id":"2a9e833e-1016-43c8-92cb-a6e8aa63f69d"} {"msg":"ds_connection connected from Ok(127.0.0.1:50212)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:02.421620418Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1531,"":"io task","client":"2","":"downstairs","session_id":"2a9e833e-1016-43c8-92cb-a6e8aa63f69d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:02.422268949Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1531,"client":"0","":"downstairs","session_id":"2a9e833e-1016-43c8-92cb-a6e8aa63f69d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:02.422313459Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1531,"client":"2","":"downstairs","session_id":"2a9e833e-1016-43c8-92cb-a6e8aa63f69d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:02.422327129Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1531,"client":"1","":"downstairs","session_id":"2a9e833e-1016-43c8-92cb-a6e8aa63f69d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:02.42258268Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1531,"session_id":"2a9e833e-1016-43c8-92cb-a6e8aa63f69d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:02.42259554Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1531,"session_id":"2a9e833e-1016-43c8-92cb-a6e8aa63f69d"} {"msg":"[0]R flush_numbers[0..12]: [148, 148, 141, 147, 151, 149, 143, 151, 150, 136, 139, 151]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:02.42261534Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1531,"":"downstairs","":"downstairs","session_id":"2a9e833e-1016-43c8-92cb-a6e8aa63f69d"} {"msg":"[0]R generation[0..12]: [94, 94, 92, 94, 96, 94, 94, 96, 95, 88, 90, 96]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:02.42262608Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1531,"":"downstairs","":"downstairs","session_id":"2a9e833e-1016-43c8-92cb-a6e8aa63f69d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:02.42263718Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1531,"":"downstairs","":"downstairs","session_id":"2a9e833e-1016-43c8-92cb-a6e8aa63f69d"} {"msg":"[1]R flush_numbers[0..12]: [148, 148, 141, 147, 151, 149, 143, 151, 150, 136, 139, 151]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:02.4226479Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1531,"":"downstairs","":"downstairs","session_id":"2a9e833e-1016-43c8-92cb-a6e8aa63f69d"} {"msg":"[1]R generation[0..12]: [94, 94, 92, 94, 96, 94, 94, 96, 95, 88, 90, 96]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:02.42265874Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1531,"":"downstairs","":"downstairs","session_id":"2a9e833e-1016-43c8-92cb-a6e8aa63f69d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:02.42266992Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1531,"":"downstairs","":"downstairs","session_id":"2a9e833e-1016-43c8-92cb-a6e8aa63f69d"} {"msg":"[2]R flush_numbers[0..12]: [148, 148, 141, 147, 151, 149, 143, 151, 150, 136, 139, 151]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:02.42268061Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1531,"":"downstairs","":"downstairs","session_id":"2a9e833e-1016-43c8-92cb-a6e8aa63f69d"} {"msg":"[2]R generation[0..12]: [94, 94, 92, 94, 96, 94, 94, 96, 95, 88, 90, 96]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:02.4226961Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1531,"":"downstairs","":"downstairs","session_id":"2a9e833e-1016-43c8-92cb-a6e8aa63f69d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:02.42270721Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1531,"":"downstairs","":"downstairs","session_id":"2a9e833e-1016-43c8-92cb-a6e8aa63f69d"} {"msg":"Max found gen is 97","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:02.42271661Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1531,"":"downstairs","":"downstairs","session_id":"2a9e833e-1016-43c8-92cb-a6e8aa63f69d"} {"msg":"Generation requested: 98 >= found:97","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:02.42272647Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1531,"":"downstairs","":"downstairs","session_id":"2a9e833e-1016-43c8-92cb-a6e8aa63f69d"} {"msg":"Next flush: 152","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:02.42273599Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1531,"":"downstairs","":"downstairs","session_id":"2a9e833e-1016-43c8-92cb-a6e8aa63f69d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:02.42274556Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1531,"":"downstairs","":"downstairs","session_id":"2a9e833e-1016-43c8-92cb-a6e8aa63f69d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:02.42275537Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1531,"session_id":"2a9e833e-1016-43c8-92cb-a6e8aa63f69d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:02.42276415Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1531,"client":"0","":"downstairs","session_id":"2a9e833e-1016-43c8-92cb-a6e8aa63f69d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:02.42277506Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1531,"client":"1","":"downstairs","session_id":"2a9e833e-1016-43c8-92cb-a6e8aa63f69d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:02.42278484Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1531,"client":"2","":"downstairs","session_id":"2a9e833e-1016-43c8-92cb-a6e8aa63f69d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:02.42279479Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1531,"session_id":"2a9e833e-1016-43c8-92cb-a6e8aa63f69d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:02.422804Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1531,"session_id":"2a9e833e-1016-43c8-92cb-a6e8aa63f69d"} {"msg":"e73d92d0-0df2-49de-94bb-b7af1c6aa4c0 is now active with session: 2a9e833e-1016-43c8-92cb-a6e8aa63f69d","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:02.42281426Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1531,"session_id":"2a9e833e-1016-43c8-92cb-a6e8aa63f69d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:02.42282298Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1531,"session_id":"2a9e833e-1016-43c8-92cb-a6e8aa63f69d"} {"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:02.42283217Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1531} {"msg":"Activated sub_volume e73d92d0-0df2-49de-94bb-b7af1c6aa4c0","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:02.42284023Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1531} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:02.42284843Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1531,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:02.423201131Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1531,"task":"crutest"} 01/30 Write block 360 len 36864 data: 15 11 11 11 13 12 11 8 8 02/30 Read block 57 len 24576 03/30 Read block 92 len 36864 04/30 Write block 170 len 20480 data: 7 7 8 8 6 05/30 Read block 448 len 40960 06/30 Write block 347 len 16384 data: 10 10 8 5 07/30 Read block 166 len 28672 08/30 Flush 09/30 Write block 102 len 28672 data: 6 8 9 8 10 9 9 10/30 Write block 274 len 8192 data: 10 11 11/30 Write block 177 len 24576 data: 5 8 5 6 6 4 12/30 Read block 168 len 20480 13/30 Read block 94 len 8192 14/30 Read block 437 len 28672 15/30 Read block 482 len 20480 16/30 Read block 123 len 40960 17/30 Write block 266 len 32768 data: 11 12 13 15 14 13 14 8 18/30 Flush 19/30 Write block 75 len 24576 data: 4 6 7 8 8 8 20/30 Read block 588 len 16384 21/30 Write block 544 len 12288 data: 9 7 6 22/30 Flush 23/30 Write block 29 len 24576 data: 9 11 11 8 6 5 24/30 Flush 25/30 Write block 258 len 4096 data: 6 26/30 Flush 27/30 Write block 8 len 20480 data: 7 4 5 5 7 28/30 Read block 399 len 20480 29/30 Read block 141 len 20480 30/30 Read block 268 len 28672 ---------------------------------------------------------------- Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Read 6 Done Done Done false 1002 Acked Read 9 Done Done Done false 1003 Acked Write 5 Done Sent Done false 1004 Acked Read 10 Done Sent Done false 1005 Acked Write 4 Done Sent Done false 1006 Acked Read 7 Done Sent Done false 1007 Acked Flush 0 Done Sent Done false 1008 Acked Write 7 Done Sent Done false 1009 Acked Write 2 Done Sent Done false 1010 Acked Write 6 Done Sent Done false 1011 Acked Read 5 Done Sent Done false 1012 Acked Read 2 Done Sent Done false 1013 Acked Read 7 Done Sent Done false 1014 Acked Read 5 Done Sent Done false 1015 Acked Read 10 Done Sent Done false 1016 Acked Write 8 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Write 6 Done Sent Done false 1019 Acked Read 4 Done Sent Done false 1020 Acked Write 3 Done Sent Done false 1021 Acked Flush 0 Done Sent Done false 1022 Acked Write 6 Done Sent Done false 1023 Acked Flush 0 Done Sent Done false 1024 Acked Write 1 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 5 Done Sent Done false 1027 Acked Read 5 Done Sent Done false 1028 Acked Read 5 Done Sent Done false 1029 Acked Read 7 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 27 0 27 Done 30 3 30 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) None Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:23:02.432666665Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1531} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:02.432854335Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1531,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 94 94 94 148 148 148 T F T <--- 1 020-039 98 94 98 155 148 155 F F F <--- 2 040-059 92 92 92 141 141 141 F F F 3 060-079 98 94 98 154 147 154 F F F <--- 4 080-099 98 96 98 154 151 154 F F F <--- 5 100-119 98 94 98 153 149 153 F F F <--- 6 120-139 94 94 94 143 143 143 F F F 7 140-159 96 96 96 151 151 151 F F F 8 160-179 98 95 98 153 150 153 F F F <--- 9 180-199 98 88 98 153 136 153 F F F <--- 10 200-219 90 90 90 139 139 139 F F F 11 220-239 96 96 96 151 151 151 F F F 12 240-259 98 94 98 156 144 156 F F F <--- 13 260-279 98 96 98 153 151 153 F F F <--- 14 280-299 96 96 96 151 151 151 F F F 15 300-319 86 86 86 134 134 134 F F F 16 320-339 90 90 90 139 139 139 F F F 17 340-359 98 94 98 152 148 152 F F F <--- 18 360-379 98 96 98 152 151 152 F T F <--- 19 380-399 96 96 96 151 151 151 F F F 20 400-419 94 94 94 145 145 145 F F F 21 420-439 96 96 96 151 151 151 F F F 22 440-459 96 96 96 151 151 151 F F F 23 460-479 96 96 96 151 151 151 F F F 24 480-499 90 90 90 139 139 139 F F F 25 500-519 95 95 95 150 150 150 F F F 26 520-539 89 89 89 138 138 138 F F F 27 540-559 98 90 98 154 139 154 F F F <--- 28 560-579 92 92 92 141 141 141 F F F 29 580-599 94 94 94 146 146 146 F F F Max gen: 98, Max flush: 156 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 49 Verifying data now No change after verify Loop: 49 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 99 99 99 157 157 157 F F F 1 020-039 98 98 98 155 155 155 F F F 2 040-059 92 92 92 141 141 141 F F F 3 060-079 98 98 98 154 154 154 F F F 4 080-099 98 98 98 154 154 154 F F F 5 100-119 98 98 98 153 153 153 F F F 6 120-139 94 94 94 143 143 143 F F F 7 140-159 96 96 96 151 151 151 F F F 8 160-179 98 98 98 153 153 153 F F F 9 180-199 98 98 98 153 153 153 F F F 10 200-219 90 90 90 139 139 139 F F F 11 220-239 96 96 96 151 151 151 F F F 12 240-259 98 98 98 156 156 156 F F F 13 260-279 98 98 98 153 153 153 F F F 14 280-299 96 96 96 151 151 151 F F F 15 300-319 86 86 86 134 134 134 F F F 16 320-339 90 90 90 139 139 139 F F F 17 340-359 98 98 98 152 152 152 F F F 18 360-379 98 98 98 152 152 152 F F F 19 380-399 96 96 96 151 151 151 F F F 20 400-419 94 94 94 145 145 145 F F F 21 420-439 96 96 96 151 151 151 F F F 22 440-459 96 96 96 151 151 151 F F F 23 460-479 96 96 96 151 151 151 F F F 24 480-499 90 90 90 139 139 139 F F F 25 500-519 95 95 95 150 150 150 F F F 26 520-539 89 89 89 138 138 138 F F F 27 540-559 98 98 98 154 154 154 F F F 28 560-579 92 92 92 141 141 141 F F F 29 580-599 94 94 94 146 146 146 F F F Max gen: 99, Max flush: 157 Begin loop 50 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:06.675456177Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1539,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:23:06.682589308Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1539,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:06.682638888Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1539,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:06.689509639Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1539,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:06.689773009Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1539} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:06.689808809Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1539} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:06.689819889Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1539} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:06.689831749Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1539} {"msg":"Crucible 22372c62-46b6-4962-bf5e-69c504892817 has session id: 8cc87930-b12f-478f-a752-96155882a9b4","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:06.69018484Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1539,"session_id":"8cc87930-b12f-478f-a752-96155882a9b4"} {"msg":"Upstairs opts: Upstairs UUID: 22372c62-46b6-4962-bf5e-69c504892817, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:06.69021454Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1539,"session_id":"8cc87930-b12f-478f-a752-96155882a9b4"} {"msg":"Crucible stats registered with UUID: 22372c62-46b6-4962-bf5e-69c504892817","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:06.69022567Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1539,"session_id":"8cc87930-b12f-478f-a752-96155882a9b4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:23:06.69023499Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1539,"session_id":"8cc87930-b12f-478f-a752-96155882a9b4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:06.69035901Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1539,"":"io task","client":"0","":"downstairs","session_id":"8cc87930-b12f-478f-a752-96155882a9b4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:06.69038907Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1539,"":"io task","client":"1","":"downstairs","session_id":"8cc87930-b12f-478f-a752-96155882a9b4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:06.69039988Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1539,"":"io task","client":"2","":"downstairs","session_id":"8cc87930-b12f-478f-a752-96155882a9b4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:06.6905728Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1539,"task":"crutest"} {"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:06.69059721Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1539} {"msg":"22372c62-46b6-4962-bf5e-69c504892817 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:06.69061359Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1539,"session_id":"8cc87930-b12f-478f-a752-96155882a9b4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:06.69062245Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1539,"client":"0","":"downstairs","session_id":"8cc87930-b12f-478f-a752-96155882a9b4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:06.69063231Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1539,"client":"1","":"downstairs","session_id":"8cc87930-b12f-478f-a752-96155882a9b4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:06.69064623Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1539,"client":"2","":"downstairs","session_id":"8cc87930-b12f-478f-a752-96155882a9b4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:06.69065761Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1539,"":"io task","client":"2","":"downstairs","session_id":"8cc87930-b12f-478f-a752-96155882a9b4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:06.6906678Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1539,"":"io task","client":"0","":"downstairs","session_id":"8cc87930-b12f-478f-a752-96155882a9b4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:06.69067799Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1539,"":"io task","client":"1","":"downstairs","session_id":"8cc87930-b12f-478f-a752-96155882a9b4"} {"msg":"ds_connection connected from Ok(127.0.0.1:35792)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:06.69075913Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1539,"":"io task","client":"2","":"downstairs","session_id":"8cc87930-b12f-478f-a752-96155882a9b4"} {"msg":"ds_connection connected from Ok(127.0.0.1:52031)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:06.690840431Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1539,"":"io task","client":"1","":"downstairs","session_id":"8cc87930-b12f-478f-a752-96155882a9b4"} {"msg":"ds_connection connected from Ok(127.0.0.1:62651)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:06.690865791Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1539,"":"io task","client":"0","":"downstairs","session_id":"8cc87930-b12f-478f-a752-96155882a9b4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:06.691425771Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1539,"client":"2","":"downstairs","session_id":"8cc87930-b12f-478f-a752-96155882a9b4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:06.691484292Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1539,"client":"1","":"downstairs","session_id":"8cc87930-b12f-478f-a752-96155882a9b4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:06.691511472Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1539,"client":"0","":"downstairs","session_id":"8cc87930-b12f-478f-a752-96155882a9b4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:06.691753222Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1539,"session_id":"8cc87930-b12f-478f-a752-96155882a9b4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:06.691777812Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1539,"session_id":"8cc87930-b12f-478f-a752-96155882a9b4"} {"msg":"[0]R flush_numbers[0..12]: [157, 155, 141, 154, 154, 153, 143, 151, 153, 153, 139, 151]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:06.691871132Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1539,"":"downstairs","":"downstairs","session_id":"8cc87930-b12f-478f-a752-96155882a9b4"} {"msg":"[0]R generation[0..12]: [99, 98, 92, 98, 98, 98, 94, 96, 98, 98, 90, 96]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:06.691882332Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1539,"":"downstairs","":"downstairs","session_id":"8cc87930-b12f-478f-a752-96155882a9b4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:06.691893592Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1539,"":"downstairs","":"downstairs","session_id":"8cc87930-b12f-478f-a752-96155882a9b4"} {"msg":"[1]R flush_numbers[0..12]: [157, 155, 141, 154, 154, 153, 143, 151, 153, 153, 139, 151]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:06.691904302Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1539,"":"downstairs","":"downstairs","session_id":"8cc87930-b12f-478f-a752-96155882a9b4"} {"msg":"[1]R generation[0..12]: [99, 98, 92, 98, 98, 98, 94, 96, 98, 98, 90, 96]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:06.691916822Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1539,"":"downstairs","":"downstairs","session_id":"8cc87930-b12f-478f-a752-96155882a9b4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:06.691927862Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1539,"":"downstairs","":"downstairs","session_id":"8cc87930-b12f-478f-a752-96155882a9b4"} {"msg":"[2]R flush_numbers[0..12]: [157, 155, 141, 154, 154, 153, 143, 151, 153, 153, 139, 151]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:06.691938682Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1539,"":"downstairs","":"downstairs","session_id":"8cc87930-b12f-478f-a752-96155882a9b4"} {"msg":"[2]R generation[0..12]: [99, 98, 92, 98, 98, 98, 94, 96, 98, 98, 90, 96]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:06.691949232Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1539,"":"downstairs","":"downstairs","session_id":"8cc87930-b12f-478f-a752-96155882a9b4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:06.691960232Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1539,"":"downstairs","":"downstairs","session_id":"8cc87930-b12f-478f-a752-96155882a9b4"} {"msg":"Max found gen is 100","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:06.691969802Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1539,"":"downstairs","":"downstairs","session_id":"8cc87930-b12f-478f-a752-96155882a9b4"} {"msg":"Generation requested: 100 >= found:100","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:06.691979832Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1539,"":"downstairs","":"downstairs","session_id":"8cc87930-b12f-478f-a752-96155882a9b4"} {"msg":"Next flush: 158","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:06.691989242Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1539,"":"downstairs","":"downstairs","session_id":"8cc87930-b12f-478f-a752-96155882a9b4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:06.691998772Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1539,"":"downstairs","":"downstairs","session_id":"8cc87930-b12f-478f-a752-96155882a9b4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:06.692008752Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1539,"session_id":"8cc87930-b12f-478f-a752-96155882a9b4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:06.692017652Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1539,"client":"0","":"downstairs","session_id":"8cc87930-b12f-478f-a752-96155882a9b4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:06.692027492Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1539,"client":"1","":"downstairs","session_id":"8cc87930-b12f-478f-a752-96155882a9b4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:06.692037382Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1539,"client":"2","":"downstairs","session_id":"8cc87930-b12f-478f-a752-96155882a9b4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:06.692047362Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1539,"session_id":"8cc87930-b12f-478f-a752-96155882a9b4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:06.692056752Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1539,"session_id":"8cc87930-b12f-478f-a752-96155882a9b4"} {"msg":"22372c62-46b6-4962-bf5e-69c504892817 is now active with session: 8cc87930-b12f-478f-a752-96155882a9b4","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:06.692068222Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1539,"session_id":"8cc87930-b12f-478f-a752-96155882a9b4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:06.692077082Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1539,"session_id":"8cc87930-b12f-478f-a752-96155882a9b4"} {"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:06.692086392Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1539} {"msg":"Activated sub_volume 22372c62-46b6-4962-bf5e-69c504892817","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:06.692094402Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1539} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:06.692102682Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1539,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:06.692384813Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1539,"task":"crutest"} 01/30 Write block 3 len 28672 data: 6 6 5 7 7 8 5 02/30 Read block 137 len 32768 03/30 Read block 466 len 8192 04/30 Write block 217 len 16384 data: 10 10 9 6 05/30 Read block 9 len 8192 06/30 Read block 44 len 12288 07/30 Read block 354 len 8192 08/30 Read block 66 len 24576 09/30 Read block 476 len 32768 10/30 Read block 73 len 8192 11/30 Read block 8 len 28672 12/30 Flush 13/30 Write block 8 len 28672 data: 9 6 6 6 8 6 7 14/30 Flush 15/30 Write block 265 len 16384 data: 13 12 13 14 16/30 Write block 208 len 32768 data: 8 9 10 10 9 7 6 7 17/30 Flush 18/30 Write block 2 len 36864 data: 4 7 7 6 8 8 10 7 7 19/30 Write block 586 len 12288 data: 8 9 9 20/30 Write block 74 len 28672 data: 6 5 7 8 9 9 9 21/30 Read block 516 len 36864 22/30 Read block 521 len 4096 23/30 Write block 308 len 32768 data: 4 5 6 5 5 5 5 4 24/30 Read block 372 len 8192 25/30 Write block 18 len 4096 data: 9 26/30 Write block 83 len 8192 data: 8 8 27/30 Write block 77 len 36864 data: 9 10 10 10 7 7 9 9 11 28/30 Read block 94 len 4096 29/30 Read block 1 len 12288 30/30 Write block 546 len 20480 data: 7 5 7 7 5 ---------------------------------------------------------------- Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Read 8 Done Done Done false 1002 Acked Read 2 Done Done Done false 1003 Acked Write 4 Done Done Done false 1004 Acked Read 2 Sent Done Done false 1005 Acked Read 3 Sent Done Done false 1006 Acked Read 2 Sent Done Done false 1007 Acked Read 6 Sent Done Done false 1008 Acked Read 8 Sent Done Done false 1009 Acked Read 2 Sent Done Done false 1010 Acked Read 7 Sent Done Done false 1011 Acked Flush 0 Sent Done Done false 1012 Acked Write 7 Sent Done Done false 1013 Acked Flush 0 Sent Done Done false 1014 Acked Write 4 Sent Done Done false 1015 Acked Write 8 Sent Done Done false 1016 Acked Flush 0 Sent Done Done false 1017 Acked Write 9 Sent Done Done false 1018 Acked Write 3 Sent Done Done false 1019 Acked Write 7 Sent Done Done false 1020 Acked Read 9 Sent Done Done false 1021 Acked Read 1 Sent Done Done false 1022 Acked Write 8 Sent Done Done false 1023 Acked Read 2 Sent Done Done false 1024 Acked Write 1 Sent Done Done false 1025 Acked Write 2 Sent Done Done false 1026 Acked Write 9 Sent Done Done false 1027 Acked Read 1 Sent Done Done false 1028 Acked Read 3 Sent Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 26 1 1 28 Done 4 29 29 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:23:06.700763376Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1539} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:06.700966296Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1539,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 99 100 100 157 159 159 T T T <--- 1 020-039 98 98 98 155 155 155 F F F 2 040-059 92 92 92 141 141 141 F F F 3 060-079 98 98 98 154 154 154 F T T <--- 4 080-099 98 98 98 154 154 154 F T T <--- 5 100-119 98 98 98 153 153 153 F F F 6 120-139 94 94 94 143 143 143 F F F 7 140-159 96 96 96 151 151 151 F F F 8 160-179 98 98 98 153 153 153 F F F 9 180-199 98 98 98 153 153 153 F F F 10 200-219 90 100 100 139 160 160 T F F <--- 11 220-239 96 100 100 151 158 158 T F F <--- 12 240-259 98 98 98 156 156 156 F F F 13 260-279 98 100 100 153 160 160 F F F <--- 14 280-299 96 96 96 151 151 151 F F F 15 300-319 86 86 86 134 134 134 F T T <--- 16 320-339 90 90 90 139 139 139 F F F 17 340-359 98 98 98 152 152 152 F F F 18 360-379 98 98 98 152 152 152 F F F 19 380-399 96 96 96 151 151 151 F F F 20 400-419 94 94 94 145 145 145 F F F 21 420-439 96 96 96 151 151 151 F F F 22 440-459 96 96 96 151 151 151 F F F 23 460-479 96 96 96 151 151 151 F F F 24 480-499 90 90 90 139 139 139 F F F 25 500-519 95 95 95 150 150 150 F F F 26 520-539 89 89 89 138 138 138 F F F 27 540-559 98 98 98 154 154 154 F T T <--- 28 560-579 92 92 92 141 141 141 F F F 29 580-599 94 94 94 146 146 146 F T T <--- Max gen: 100, Max flush: 160 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 50 Verifying data now No change after verify Loop: 50 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 101 101 101 161 161 161 F F F 1 020-039 98 98 98 155 155 155 F F F 2 040-059 92 92 92 141 141 141 F F F 3 060-079 101 101 101 161 161 161 F F F 4 080-099 101 101 101 161 161 161 F F F 5 100-119 98 98 98 153 153 153 F F F 6 120-139 94 94 94 143 143 143 F F F 7 140-159 96 96 96 151 151 151 F F F 8 160-179 98 98 98 153 153 153 F F F 9 180-199 98 98 98 153 153 153 F F F 10 200-219 100 100 100 160 160 160 F F F 11 220-239 100 100 100 158 158 158 F F F 12 240-259 98 98 98 156 156 156 F F F 13 260-279 100 100 100 160 160 160 F F F 14 280-299 96 96 96 151 151 151 F F F 15 300-319 101 101 101 161 161 161 F F F 16 320-339 90 90 90 139 139 139 F F F 17 340-359 98 98 98 152 152 152 F F F 18 360-379 98 98 98 152 152 152 F F F 19 380-399 96 96 96 151 151 151 F F F 20 400-419 94 94 94 145 145 145 F F F 21 420-439 96 96 96 151 151 151 F F F 22 440-459 96 96 96 151 151 151 F F F 23 460-479 96 96 96 151 151 151 F F F 24 480-499 90 90 90 139 139 139 F F F 25 500-519 95 95 95 150 150 150 F F F 26 520-539 89 89 89 138 138 138 F F F 27 540-559 101 101 101 161 161 161 F F F 28 560-579 92 92 92 141 141 141 F F F 29 580-599 101 101 101 161 161 161 F F F Max gen: 101, Max flush: 161 Begin loop 51 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:10.948535485Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1547,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:10.956713147Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1547,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:10.957011728Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1547} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:10.957048748Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1547} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:10.957058188Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1547} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:10.957070088Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1547} {"msg":"Crucible ccdebbe6-6b71-4013-84e8-4e7818bc5e3b has session id: f1b96e39-fcb3-4b00-81cb-8b8568d7708a","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:10.957408338Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1547,"session_id":"f1b96e39-fcb3-4b00-81cb-8b8568d7708a"} {"msg":"Upstairs opts: Upstairs UUID: ccdebbe6-6b71-4013-84e8-4e7818bc5e3b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:10.957437808Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1547,"session_id":"f1b96e39-fcb3-4b00-81cb-8b8568d7708a"} {"msg":"Crucible stats registered with UUID: ccdebbe6-6b71-4013-84e8-4e7818bc5e3b","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:10.957451188Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1547,"session_id":"f1b96e39-fcb3-4b00-81cb-8b8568d7708a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:23:10.957460579Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1547,"session_id":"f1b96e39-fcb3-4b00-81cb-8b8568d7708a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:10.957572179Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1547,"":"io task","client":"1","":"downstairs","session_id":"f1b96e39-fcb3-4b00-81cb-8b8568d7708a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:10.957590139Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1547,"":"io task","client":"0","":"downstairs","session_id":"f1b96e39-fcb3-4b00-81cb-8b8568d7708a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:10.957600319Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1547,"":"io task","client":"2","":"downstairs","session_id":"f1b96e39-fcb3-4b00-81cb-8b8568d7708a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:10.957749709Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1547,"task":"crutest"} {"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:10.957764159Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1547} {"msg":"ccdebbe6-6b71-4013-84e8-4e7818bc5e3b active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:10.957800069Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1547,"session_id":"f1b96e39-fcb3-4b00-81cb-8b8568d7708a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:10.957809469Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1547,"client":"0","":"downstairs","session_id":"f1b96e39-fcb3-4b00-81cb-8b8568d7708a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:10.957819259Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1547,"client":"1","":"downstairs","session_id":"f1b96e39-fcb3-4b00-81cb-8b8568d7708a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:10.957829019Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1547,"client":"2","":"downstairs","session_id":"f1b96e39-fcb3-4b00-81cb-8b8568d7708a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:10.957838649Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1547,"":"io task","client":"2","":"downstairs","session_id":"f1b96e39-fcb3-4b00-81cb-8b8568d7708a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:10.957848559Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1547,"":"io task","client":"0","":"downstairs","session_id":"f1b96e39-fcb3-4b00-81cb-8b8568d7708a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:10.957858599Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1547,"":"io task","client":"1","":"downstairs","session_id":"f1b96e39-fcb3-4b00-81cb-8b8568d7708a"} {"msg":"ds_connection connected from Ok(127.0.0.1:56041)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:10.957917709Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1547,"":"io task","client":"2","":"downstairs","session_id":"f1b96e39-fcb3-4b00-81cb-8b8568d7708a"} {"msg":"ds_connection connected from Ok(127.0.0.1:59096)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:10.957928669Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1547,"":"io task","client":"1","":"downstairs","session_id":"f1b96e39-fcb3-4b00-81cb-8b8568d7708a"} {"msg":"ds_connection connected from Ok(127.0.0.1:60279)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:10.957998119Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1547,"":"io task","client":"0","":"downstairs","session_id":"f1b96e39-fcb3-4b00-81cb-8b8568d7708a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:10.9585334Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1547,"client":"0","":"downstairs","session_id":"f1b96e39-fcb3-4b00-81cb-8b8568d7708a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:10.95861305Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1547,"client":"2","":"downstairs","session_id":"f1b96e39-fcb3-4b00-81cb-8b8568d7708a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:10.95863347Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1547,"client":"1","":"downstairs","session_id":"f1b96e39-fcb3-4b00-81cb-8b8568d7708a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:10.958871231Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1547,"session_id":"f1b96e39-fcb3-4b00-81cb-8b8568d7708a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:10.958938201Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1547,"session_id":"f1b96e39-fcb3-4b00-81cb-8b8568d7708a"} {"msg":"[0]R flush_numbers[0..12]: [161, 155, 141, 161, 161, 153, 143, 151, 153, 153, 160, 158]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:10.958958671Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1547,"":"downstairs","":"downstairs","session_id":"f1b96e39-fcb3-4b00-81cb-8b8568d7708a"} {"msg":"[0]R generation[0..12]: [101, 98, 92, 101, 101, 98, 94, 96, 98, 98, 100, 100]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:10.958969931Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1547,"":"downstairs","":"downstairs","session_id":"f1b96e39-fcb3-4b00-81cb-8b8568d7708a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:10.958981331Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1547,"":"downstairs","":"downstairs","session_id":"f1b96e39-fcb3-4b00-81cb-8b8568d7708a"} {"msg":"[1]R flush_numbers[0..12]: [161, 155, 141, 161, 161, 153, 143, 151, 153, 153, 160, 158]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:10.958992201Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1547,"":"downstairs","":"downstairs","session_id":"f1b96e39-fcb3-4b00-81cb-8b8568d7708a"} {"msg":"[1]R generation[0..12]: [101, 98, 92, 101, 101, 98, 94, 96, 98, 98, 100, 100]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:10.959002891Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1547,"":"downstairs","":"downstairs","session_id":"f1b96e39-fcb3-4b00-81cb-8b8568d7708a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:10.959014091Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1547,"":"downstairs","":"downstairs","session_id":"f1b96e39-fcb3-4b00-81cb-8b8568d7708a"} {"msg":"[2]R flush_numbers[0..12]: [161, 155, 141, 161, 161, 153, 143, 151, 153, 153, 160, 158]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:10.959024841Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1547,"":"downstairs","":"downstairs","session_id":"f1b96e39-fcb3-4b00-81cb-8b8568d7708a"} {"msg":"[2]R generation[0..12]: [101, 98, 92, 101, 101, 98, 94, 96, 98, 98, 100, 100]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:10.959035371Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1547,"":"downstairs","":"downstairs","session_id":"f1b96e39-fcb3-4b00-81cb-8b8568d7708a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:10.959046391Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1547,"":"downstairs","":"downstairs","session_id":"f1b96e39-fcb3-4b00-81cb-8b8568d7708a"} {"msg":"Max found gen is 102","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:10.959058101Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1547,"":"downstairs","":"downstairs","session_id":"f1b96e39-fcb3-4b00-81cb-8b8568d7708a"} {"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:10.959068081Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1547,"":"downstairs","":"downstairs","session_id":"f1b96e39-fcb3-4b00-81cb-8b8568d7708a"} {"msg":"Next flush: 162","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:10.959077511Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1547,"":"downstairs","":"downstairs","session_id":"f1b96e39-fcb3-4b00-81cb-8b8568d7708a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:10.959086871Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1547,"":"downstairs","":"downstairs","session_id":"f1b96e39-fcb3-4b00-81cb-8b8568d7708a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:10.959097001Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1547,"session_id":"f1b96e39-fcb3-4b00-81cb-8b8568d7708a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:10.959105991Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1547,"client":"0","":"downstairs","session_id":"f1b96e39-fcb3-4b00-81cb-8b8568d7708a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:10.959116131Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1547,"client":"1","":"downstairs","session_id":"f1b96e39-fcb3-4b00-81cb-8b8568d7708a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:10.959126061Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1547,"client":"2","":"downstairs","session_id":"f1b96e39-fcb3-4b00-81cb-8b8568d7708a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:10.959136431Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1547,"session_id":"f1b96e39-fcb3-4b00-81cb-8b8568d7708a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:10.959145901Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1547,"session_id":"f1b96e39-fcb3-4b00-81cb-8b8568d7708a"} {"msg":"ccdebbe6-6b71-4013-84e8-4e7818bc5e3b is now active with session: f1b96e39-fcb3-4b00-81cb-8b8568d7708a","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:10.959156081Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1547,"session_id":"f1b96e39-fcb3-4b00-81cb-8b8568d7708a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:10.959164991Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1547,"session_id":"f1b96e39-fcb3-4b00-81cb-8b8568d7708a"} {"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:10.959174151Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1547} {"msg":"Activated sub_volume ccdebbe6-6b71-4013-84e8-4e7818bc5e3b","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:10.959182181Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1547} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:10.959190551Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1547,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:10.959512412Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1547,"task":"crutest"} 01/30 Write block 433 len 28672 data: 6 5 4 4 7 7 7 02/30 Read block 235 len 28672 03/30 Read block 522 len 28672 04/30 Read block 177 len 36864 05/30 Flush 06/30 Write block 395 len 32768 data: 8 6 6 7 8 7 5 6 07/30 Read block 280 len 40960 08/30 Read block 182 len 24576 09/30 Read block 25 len 16384 10/30 Read block 551 len 4096 11/30 Write block 106 len 12288 data: 11 10 10 12/30 Read block 567 len 20480 13/30 Read block 117 len 40960 14/30 Read block 48 len 32768 15/30 Write block 30 len 4096 data: 12 16/30 Write block 455 len 8192 data: 8 9 17/30 Write block 134 len 32768 data: 5 7 6 6 9 12 11 9 18/30 Read block 364 len 40960 19/30 Write block 258 len 8192 data: 7 8 20/30 Read block 469 len 36864 21/30 Read block 592 len 12288 22/30 Read block 201 len 4096 23/30 Read block 164 len 8192 24/30 Read block 69 len 16384 25/30 Write block 476 len 12288 data: 7 8 8 26/30 Read block 146 len 16384 27/30 Write block 329 len 20480 data: 7 9 10 8 6 28/30 Read block 41 len 24576 29/30 Write block 23 len 28672 data: 10 9 8 8 10 10 10 30/30 Write block 39 len 4096 data: 7 ---------------------------------------------------------------- Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Sent Done false 1001 Acked Read 7 Done Sent Done false 1002 Acked Read 7 Done Sent Done false 1003 Acked Read 9 Done Sent Done false 1004 Acked Flush 0 Done Sent Done false 1005 Acked Write 8 Done Sent Done false 1006 Acked Read 10 Done Sent Done false 1007 Acked Read 6 Done Sent Done false 1008 Acked Read 4 Done Sent Done false 1009 Acked Read 1 Done Sent Done false 1010 Acked Write 3 Done Sent Done false 1011 Acked Read 5 Done Sent Done false 1012 Acked Read 10 Done Sent Done false 1013 Acked Read 8 Done Sent Done false 1014 Acked Write 1 Done Sent Done false 1015 Acked Write 2 Done Sent Done false 1016 Acked Write 8 Done Sent Done false 1017 Acked Read 10 Done Sent Done false 1018 Acked Write 2 Done Sent Done false 1019 Acked Read 9 Done Sent Done false 1020 Acked Read 3 Done Sent Done false 1021 Acked Read 1 Done Sent Done false 1022 Acked Read 2 Done Sent Done false 1023 Acked Read 4 Done Sent Done false 1024 Acked Write 3 Done Sent Done false 1025 Acked Read 4 Done Sent Done false 1026 Acked Write 5 Done Sent Done false 1027 Acked Read 6 Done Sent Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 30 2 34 Done 28 0 28 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1004)) None Some(JobId(1004)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:23:10.968062515Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1547} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:10.968211695Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1547,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 101 101 101 161 161 161 F F F 1 020-039 98 98 98 155 155 155 T F T <--- 2 040-059 92 92 92 141 141 141 F F F 3 060-079 101 101 101 161 161 161 F F F 4 080-099 101 101 101 161 161 161 F F F 5 100-119 98 98 98 153 153 153 T F T <--- 6 120-139 94 94 94 143 143 143 T F T <--- 7 140-159 96 96 96 151 151 151 T F T <--- 8 160-179 98 98 98 153 153 153 F F F 9 180-199 98 98 98 153 153 153 F F F 10 200-219 100 100 100 160 160 160 F F F 11 220-239 100 100 100 158 158 158 F F F 12 240-259 98 98 98 156 156 156 T F T <--- 13 260-279 100 100 100 160 160 160 F F F 14 280-299 96 96 96 151 151 151 F F F 15 300-319 101 101 101 161 161 161 F F F 16 320-339 90 90 90 139 139 139 T F T <--- 17 340-359 98 98 98 152 152 152 F F F 18 360-379 98 98 98 152 152 152 F F F 19 380-399 96 96 96 151 151 151 T F T <--- 20 400-419 94 94 94 145 145 145 T F T <--- 21 420-439 102 96 102 162 151 162 F F F <--- 22 440-459 96 96 96 151 151 151 T F T <--- 23 460-479 96 96 96 151 151 151 T F T <--- 24 480-499 90 90 90 139 139 139 F F F 25 500-519 95 95 95 150 150 150 F F F 26 520-539 89 89 89 138 138 138 F F F 27 540-559 101 101 101 161 161 161 F F F 28 560-579 92 92 92 141 141 141 F F F 29 580-599 101 101 101 161 161 161 F F F Max gen: 102, Max flush: 162 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 51 Verifying data now No change after verify Loop: 51 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 101 101 101 161 161 161 F F F 1 020-039 103 103 103 163 163 163 F F F 2 040-059 92 92 92 141 141 141 F F F 3 060-079 101 101 101 161 161 161 F F F 4 080-099 101 101 101 161 161 161 F F F 5 100-119 103 103 103 163 163 163 F F F 6 120-139 103 103 103 163 163 163 F F F 7 140-159 103 103 103 163 163 163 F F F 8 160-179 98 98 98 153 153 153 F F F 9 180-199 98 98 98 153 153 153 F F F 10 200-219 100 100 100 160 160 160 F F F 11 220-239 100 100 100 158 158 158 F F F 12 240-259 103 103 103 163 163 163 F F F 13 260-279 100 100 100 160 160 160 F F F 14 280-299 96 96 96 151 151 151 F F F 15 300-319 101 101 101 161 161 161 F F F 16 320-339 103 103 103 163 163 163 F F F 17 340-359 98 98 98 152 152 152 F F F 18 360-379 98 98 98 152 152 152 F F F 19 380-399 103 103 103 163 163 163 F F F 20 400-419 103 103 103 163 163 163 F F F 21 420-439 102 102 102 162 162 162 F F F 22 440-459 103 103 103 163 163 163 F F F 23 460-479 103 103 103 163 163 163 F F F 24 480-499 90 90 90 139 139 139 F F F 25 500-519 95 95 95 150 150 150 F F F 26 520-539 89 89 89 138 138 138 F F F 27 540-559 101 101 101 161 161 161 F F F 28 560-579 92 92 92 141 141 141 F F F 29 580-599 101 101 101 161 161 161 F F F Max gen: 103, Max flush: 163 Begin loop 52 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:15.220920792Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1555,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:23:15.228034043Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1555,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:15.228084183Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1555,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:15.234992313Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1555,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:15.235259324Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1555} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:15.235294724Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1555} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:15.235303884Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1555} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:15.235315674Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1555} {"msg":"Crucible 5b2c72fa-91bd-4d62-a0af-9305658ca6ad has session id: 9446e35a-1230-452d-be2a-436a00a736cf","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:15.235662774Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1555,"session_id":"9446e35a-1230-452d-be2a-436a00a736cf"} {"msg":"Upstairs opts: Upstairs UUID: 5b2c72fa-91bd-4d62-a0af-9305658ca6ad, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:15.235688454Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1555,"session_id":"9446e35a-1230-452d-be2a-436a00a736cf"} {"msg":"Crucible stats registered with UUID: 5b2c72fa-91bd-4d62-a0af-9305658ca6ad","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:15.235698365Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1555,"session_id":"9446e35a-1230-452d-be2a-436a00a736cf"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:23:15.235707365Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1555,"session_id":"9446e35a-1230-452d-be2a-436a00a736cf"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:15.235836145Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1555,"":"io task","client":"2","":"downstairs","session_id":"9446e35a-1230-452d-be2a-436a00a736cf"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:15.235853535Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1555,"":"io task","client":"0","":"downstairs","session_id":"9446e35a-1230-452d-be2a-436a00a736cf"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:15.235865485Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1555,"":"io task","client":"1","":"downstairs","session_id":"9446e35a-1230-452d-be2a-436a00a736cf"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:15.236027655Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1555,"task":"crutest"} {"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:15.236046735Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1555} {"msg":"5b2c72fa-91bd-4d62-a0af-9305658ca6ad active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:15.236063585Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1555,"session_id":"9446e35a-1230-452d-be2a-436a00a736cf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:15.236072465Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1555,"client":"0","":"downstairs","session_id":"9446e35a-1230-452d-be2a-436a00a736cf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:15.236082325Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1555,"client":"1","":"downstairs","session_id":"9446e35a-1230-452d-be2a-436a00a736cf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:15.236092085Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1555,"client":"2","":"downstairs","session_id":"9446e35a-1230-452d-be2a-436a00a736cf"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:15.236101795Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1555,"":"io task","client":"2","":"downstairs","session_id":"9446e35a-1230-452d-be2a-436a00a736cf"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:15.236111805Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1555,"":"io task","client":"0","":"downstairs","session_id":"9446e35a-1230-452d-be2a-436a00a736cf"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:15.236121935Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1555,"":"io task","client":"1","":"downstairs","session_id":"9446e35a-1230-452d-be2a-436a00a736cf"} {"msg":"ds_connection connected from Ok(127.0.0.1:37268)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:15.236160225Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1555,"":"io task","client":"1","":"downstairs","session_id":"9446e35a-1230-452d-be2a-436a00a736cf"} {"msg":"ds_connection connected from Ok(127.0.0.1:35768)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:15.236171405Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1555,"":"io task","client":"2","":"downstairs","session_id":"9446e35a-1230-452d-be2a-436a00a736cf"} {"msg":"ds_connection connected from Ok(127.0.0.1:58162)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:15.236337515Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1555,"":"io task","client":"0","":"downstairs","session_id":"9446e35a-1230-452d-be2a-436a00a736cf"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:15.236793266Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1555,"client":"1","":"downstairs","session_id":"9446e35a-1230-452d-be2a-436a00a736cf"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:15.236843466Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1555,"client":"2","":"downstairs","session_id":"9446e35a-1230-452d-be2a-436a00a736cf"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:15.236896096Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1555,"client":"0","":"downstairs","session_id":"9446e35a-1230-452d-be2a-436a00a736cf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:15.237100727Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1555,"session_id":"9446e35a-1230-452d-be2a-436a00a736cf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:15.237116787Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1555,"session_id":"9446e35a-1230-452d-be2a-436a00a736cf"} {"msg":"[0]R flush_numbers[0..12]: [161, 163, 141, 161, 161, 163, 163, 163, 153, 153, 160, 158]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:15.237173067Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1555,"":"downstairs","":"downstairs","session_id":"9446e35a-1230-452d-be2a-436a00a736cf"} {"msg":"[0]R generation[0..12]: [101, 103, 92, 101, 101, 103, 103, 103, 98, 98, 100, 100]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:15.237184417Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1555,"":"downstairs","":"downstairs","session_id":"9446e35a-1230-452d-be2a-436a00a736cf"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:15.237195727Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1555,"":"downstairs","":"downstairs","session_id":"9446e35a-1230-452d-be2a-436a00a736cf"} {"msg":"[1]R flush_numbers[0..12]: [161, 163, 141, 161, 161, 163, 163, 163, 153, 153, 160, 158]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:15.237206717Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1555,"":"downstairs","":"downstairs","session_id":"9446e35a-1230-452d-be2a-436a00a736cf"} {"msg":"[1]R generation[0..12]: [101, 103, 92, 101, 101, 103, 103, 103, 98, 98, 100, 100]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:15.237217517Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1555,"":"downstairs","":"downstairs","session_id":"9446e35a-1230-452d-be2a-436a00a736cf"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:15.237228517Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1555,"":"downstairs","":"downstairs","session_id":"9446e35a-1230-452d-be2a-436a00a736cf"} {"msg":"[2]R flush_numbers[0..12]: [161, 163, 141, 161, 161, 163, 163, 163, 153, 153, 160, 158]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:15.237239317Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1555,"":"downstairs","":"downstairs","session_id":"9446e35a-1230-452d-be2a-436a00a736cf"} {"msg":"[2]R generation[0..12]: [101, 103, 92, 101, 101, 103, 103, 103, 98, 98, 100, 100]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:15.237249897Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1555,"":"downstairs","":"downstairs","session_id":"9446e35a-1230-452d-be2a-436a00a736cf"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:15.237260997Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1555,"":"downstairs","":"downstairs","session_id":"9446e35a-1230-452d-be2a-436a00a736cf"} {"msg":"Max found gen is 104","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:15.237270607Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1555,"":"downstairs","":"downstairs","session_id":"9446e35a-1230-452d-be2a-436a00a736cf"} {"msg":"Generation requested: 104 >= found:104","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:15.237280437Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1555,"":"downstairs","":"downstairs","session_id":"9446e35a-1230-452d-be2a-436a00a736cf"} {"msg":"Next flush: 164","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:15.237289937Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1555,"":"downstairs","":"downstairs","session_id":"9446e35a-1230-452d-be2a-436a00a736cf"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:15.237301197Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1555,"":"downstairs","":"downstairs","session_id":"9446e35a-1230-452d-be2a-436a00a736cf"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:15.237311127Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1555,"session_id":"9446e35a-1230-452d-be2a-436a00a736cf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:15.237320197Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1555,"client":"0","":"downstairs","session_id":"9446e35a-1230-452d-be2a-436a00a736cf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:15.237329947Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1555,"client":"1","":"downstairs","session_id":"9446e35a-1230-452d-be2a-436a00a736cf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:15.237339647Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1555,"client":"2","":"downstairs","session_id":"9446e35a-1230-452d-be2a-436a00a736cf"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:15.237349517Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1555,"session_id":"9446e35a-1230-452d-be2a-436a00a736cf"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:15.237358837Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1555,"session_id":"9446e35a-1230-452d-be2a-436a00a736cf"} {"msg":"5b2c72fa-91bd-4d62-a0af-9305658ca6ad is now active with session: 9446e35a-1230-452d-be2a-436a00a736cf","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:15.237369047Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1555,"session_id":"9446e35a-1230-452d-be2a-436a00a736cf"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:15.237377917Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1555,"session_id":"9446e35a-1230-452d-be2a-436a00a736cf"} {"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:15.237387267Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1555} {"msg":"Activated sub_volume 5b2c72fa-91bd-4d62-a0af-9305658ca6ad","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:15.237395387Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1555} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:15.237403547Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1555,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:15.237707928Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1555,"task":"crutest"} 01/30 Write block 54 len 40960 data: 8 9 7 8 9 8 8 12 12 11 02/30 Write block 227 len 8192 data: 5 7 03/30 Flush 04/30 Write block 123 len 16384 data: 7 6 7 6 05/30 Read block 458 len 36864 06/30 Read block 342 len 12288 07/30 Read block 493 len 36864 08/30 Write block 575 len 12288 data: 10 10 10 09/30 Read block 81 len 32768 10/30 Write block 153 len 40960 data: 8 11 10 9 8 11 9 10 10 6 11/30 Read block 341 len 24576 12/30 Read block 301 len 12288 13/30 Write block 297 len 40960 data: 8 10 11 10 12 10 9 9 8 8 14/30 Read block 71 len 28672 15/30 Write block 469 len 40960 data: 8 7 7 5 5 6 6 8 9 9 16/30 Read block 61 len 8192 17/30 Write block 75 len 16384 data: 6 8 10 11 18/30 Read block 279 len 24576 19/30 Write block 82 len 40960 data: 8 10 10 12 11 10 9 10 9 8 20/30 Read block 360 len 40960 21/30 Flush 22/30 Write block 519 len 12288 data: 8 8 8 23/30 Flush 24/30 Write block 478 len 4096 data: 10 25/30 Write block 446 len 36864 data: 5 7 7 8 7 6 5 5 6 26/30 Read block 347 len 40960 27/30 Write block 357 len 36864 data: 10 10 11 16 12 12 12 14 13 28/30 Read block 424 len 28672 29/30 Write block 412 len 32768 data: 6 5 6 5 7 6 6 5 30/30 Write block 573 len 28672 data: 10 10 11 11 11 12 9 ---------------------------------------------------------------- Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1003 Acked Write 4 Done Done Done false 1004 Acked Read 9 Sent Done Done false 1005 Acked Read 3 Sent Done Done false 1006 Acked Read 9 Sent Done Done false 1007 Acked Write 3 Sent Done Done false 1008 Acked Read 8 Sent Done Done false 1009 Acked Write 10 Sent Done Done false 1010 Acked Read 6 Sent Done Done false 1011 Acked Read 3 Sent Done Done false 1012 Acked Write 10 Sent Done Done false 1013 Acked Read 7 Sent Done Done false 1014 Acked Write 10 Sent Done Done false 1015 Acked Read 2 Sent Done Done false 1016 Acked Write 4 Sent Done Done false 1017 Acked Read 6 Sent Done Done false 1018 Acked Write 10 Sent Done Done false 1019 Acked Read 10 Sent Done Done false 1020 Acked Flush 0 Sent Done Done false 1021 Acked Write 3 Sent Done Done false 1022 Acked Flush 0 Sent Done Done false 1023 Acked Write 1 Sent Done Done false 1024 Acked Write 9 Sent Done Done false 1025 Acked Read 10 Sent Done Done false 1026 Acked Write 9 Sent Done Done false 1027 Acked Read 7 Sent Done Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 26 2 2 30 Done 1 25 25 51 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1002)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:23:15.247990563Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1555} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:15.248155313Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1555,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 101 101 101 161 161 161 F F F 1 020-039 103 103 103 163 163 163 F F F 2 040-059 104 104 104 164 164 164 F F F 3 060-079 104 104 104 164 165 165 F F F <--- 4 080-099 101 104 104 161 165 165 F F F <--- 5 100-119 103 103 103 163 163 163 F F F 6 120-139 103 104 104 163 165 165 T F F <--- 7 140-159 103 104 104 163 165 165 F F F <--- 8 160-179 98 104 104 153 165 165 F F F <--- 9 180-199 98 98 98 153 153 153 F F F 10 200-219 100 100 100 160 160 160 F F F 11 220-239 104 104 104 164 164 164 F F F 12 240-259 103 103 103 163 163 163 F F F 13 260-279 100 100 100 160 160 160 F F F 14 280-299 96 104 104 151 165 165 F F F <--- 15 300-319 101 104 104 161 165 165 F F F <--- 16 320-339 103 103 103 163 163 163 F F F 17 340-359 98 98 98 152 152 152 F T T <--- 18 360-379 98 98 98 152 152 152 F T T <--- 19 380-399 103 103 103 163 163 163 F F F 20 400-419 103 103 103 163 163 163 F T T <--- 21 420-439 102 102 102 162 162 162 F F F 22 440-459 103 103 103 163 163 163 F T T <--- 23 460-479 103 104 104 163 165 165 F T T <--- 24 480-499 90 90 90 139 139 139 F F F 25 500-519 95 104 104 150 166 166 F F F <--- 26 520-539 89 104 104 138 166 166 F F F <--- 27 540-559 101 101 101 161 161 161 F F F 28 560-579 92 104 104 141 165 165 F T T <--- 29 580-599 101 101 101 161 161 161 F F F Max gen: 104, Max flush: 166 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 52 Verifying data now No change after verify Loop: 52 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 101 101 101 161 161 161 F F F 1 020-039 103 103 103 163 163 163 F F F 2 040-059 104 104 104 164 164 164 F F F 3 060-079 104 104 104 165 165 165 F F F 4 080-099 104 104 104 165 165 165 F F F 5 100-119 103 103 103 163 163 163 F F F 6 120-139 104 104 104 165 165 165 F F F 7 140-159 104 104 104 165 165 165 F F F 8 160-179 104 104 104 165 165 165 F F F 9 180-199 98 98 98 153 153 153 F F F 10 200-219 100 100 100 160 160 160 F F F 11 220-239 104 104 104 164 164 164 F F F 12 240-259 103 103 103 163 163 163 F F F 13 260-279 100 100 100 160 160 160 F F F 14 280-299 104 104 104 165 165 165 F F F 15 300-319 104 104 104 165 165 165 F F F 16 320-339 103 103 103 163 163 163 F F F 17 340-359 105 105 105 167 167 167 F F F 18 360-379 105 105 105 167 167 167 F F F 19 380-399 103 103 103 163 163 163 F F F 20 400-419 105 105 105 167 167 167 F F F 21 420-439 102 102 102 162 162 162 F F F 22 440-459 105 105 105 167 167 167 F F F 23 460-479 105 105 105 167 167 167 F F F 24 480-499 90 90 90 139 139 139 F F F 25 500-519 104 104 104 166 166 166 F F F 26 520-539 104 104 104 166 166 166 F F F 27 540-559 101 101 101 161 161 161 F F F 28 560-579 105 105 105 167 167 167 F F F 29 580-599 101 101 101 161 161 161 F F F Max gen: 105, Max flush: 167 Begin loop 53 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:19.510790336Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1563,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:19.519030358Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1563,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:19.519323399Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1563} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:19.519353899Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1563} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:19.519365809Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1563} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:19.519377819Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1563} {"msg":"Crucible 9b839dc3-a36e-412f-ab00-db9b2e0fb06f has session id: 39839b33-e7c1-4cc6-b0d9-9db19d562de2","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:19.519706149Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1563,"session_id":"39839b33-e7c1-4cc6-b0d9-9db19d562de2"} {"msg":"Upstairs opts: Upstairs UUID: 9b839dc3-a36e-412f-ab00-db9b2e0fb06f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:19.519734629Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1563,"session_id":"39839b33-e7c1-4cc6-b0d9-9db19d562de2"} {"msg":"Crucible stats registered with UUID: 9b839dc3-a36e-412f-ab00-db9b2e0fb06f","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:19.519745559Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1563,"session_id":"39839b33-e7c1-4cc6-b0d9-9db19d562de2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:23:19.519754859Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1563,"session_id":"39839b33-e7c1-4cc6-b0d9-9db19d562de2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:19.519883589Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1563,"":"io task","client":"2","":"downstairs","session_id":"39839b33-e7c1-4cc6-b0d9-9db19d562de2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:19.51990164Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1563,"":"io task","client":"0","":"downstairs","session_id":"39839b33-e7c1-4cc6-b0d9-9db19d562de2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:19.51991194Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1563,"":"io task","client":"1","":"downstairs","session_id":"39839b33-e7c1-4cc6-b0d9-9db19d562de2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:19.52007805Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1563,"task":"crutest"} {"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:19.52009698Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1563} {"msg":"9b839dc3-a36e-412f-ab00-db9b2e0fb06f active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:19.52013511Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1563,"session_id":"39839b33-e7c1-4cc6-b0d9-9db19d562de2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:19.5201445Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1563,"client":"0","":"downstairs","session_id":"39839b33-e7c1-4cc6-b0d9-9db19d562de2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:19.52015464Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1563,"client":"1","":"downstairs","session_id":"39839b33-e7c1-4cc6-b0d9-9db19d562de2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:19.52016445Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1563,"client":"2","":"downstairs","session_id":"39839b33-e7c1-4cc6-b0d9-9db19d562de2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:19.52017631Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1563,"":"io task","client":"2","":"downstairs","session_id":"39839b33-e7c1-4cc6-b0d9-9db19d562de2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:19.520199Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1563,"":"io task","client":"0","":"downstairs","session_id":"39839b33-e7c1-4cc6-b0d9-9db19d562de2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:19.52021346Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1563,"":"io task","client":"1","":"downstairs","session_id":"39839b33-e7c1-4cc6-b0d9-9db19d562de2"} {"msg":"ds_connection connected from Ok(127.0.0.1:55005)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:19.52028621Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1563,"":"io task","client":"0","":"downstairs","session_id":"39839b33-e7c1-4cc6-b0d9-9db19d562de2"} {"msg":"ds_connection connected from Ok(127.0.0.1:53762)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:19.52029822Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1563,"":"io task","client":"2","":"downstairs","session_id":"39839b33-e7c1-4cc6-b0d9-9db19d562de2"} {"msg":"ds_connection connected from Ok(127.0.0.1:53900)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:19.52042675Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1563,"":"io task","client":"1","":"downstairs","session_id":"39839b33-e7c1-4cc6-b0d9-9db19d562de2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:19.520909921Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1563,"client":"0","":"downstairs","session_id":"39839b33-e7c1-4cc6-b0d9-9db19d562de2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:19.520935691Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1563,"client":"1","":"downstairs","session_id":"39839b33-e7c1-4cc6-b0d9-9db19d562de2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:19.521026661Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1563,"client":"2","":"downstairs","session_id":"39839b33-e7c1-4cc6-b0d9-9db19d562de2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:19.521189391Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1563,"session_id":"39839b33-e7c1-4cc6-b0d9-9db19d562de2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:19.521247092Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1563,"session_id":"39839b33-e7c1-4cc6-b0d9-9db19d562de2"} {"msg":"[0]R flush_numbers[0..12]: [161, 163, 164, 165, 165, 163, 165, 165, 165, 153, 160, 164]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:19.521297662Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1563,"":"downstairs","":"downstairs","session_id":"39839b33-e7c1-4cc6-b0d9-9db19d562de2"} {"msg":"[0]R generation[0..12]: [101, 103, 104, 104, 104, 103, 104, 104, 104, 98, 100, 104]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:19.521309172Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1563,"":"downstairs","":"downstairs","session_id":"39839b33-e7c1-4cc6-b0d9-9db19d562de2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:19.521320242Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1563,"":"downstairs","":"downstairs","session_id":"39839b33-e7c1-4cc6-b0d9-9db19d562de2"} {"msg":"[1]R flush_numbers[0..12]: [161, 163, 164, 165, 165, 163, 165, 165, 165, 153, 160, 164]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:19.521332982Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1563,"":"downstairs","":"downstairs","session_id":"39839b33-e7c1-4cc6-b0d9-9db19d562de2"} {"msg":"[1]R generation[0..12]: [101, 103, 104, 104, 104, 103, 104, 104, 104, 98, 100, 104]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:19.521343752Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1563,"":"downstairs","":"downstairs","session_id":"39839b33-e7c1-4cc6-b0d9-9db19d562de2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:19.521355002Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1563,"":"downstairs","":"downstairs","session_id":"39839b33-e7c1-4cc6-b0d9-9db19d562de2"} {"msg":"[2]R flush_numbers[0..12]: [161, 163, 164, 165, 165, 163, 165, 165, 165, 153, 160, 164]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:19.521365752Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1563,"":"downstairs","":"downstairs","session_id":"39839b33-e7c1-4cc6-b0d9-9db19d562de2"} {"msg":"[2]R generation[0..12]: [101, 103, 104, 104, 104, 103, 104, 104, 104, 98, 100, 104]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:19.521376402Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1563,"":"downstairs","":"downstairs","session_id":"39839b33-e7c1-4cc6-b0d9-9db19d562de2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:19.521387462Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1563,"":"downstairs","":"downstairs","session_id":"39839b33-e7c1-4cc6-b0d9-9db19d562de2"} {"msg":"Max found gen is 106","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:19.521396932Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1563,"":"downstairs","":"downstairs","session_id":"39839b33-e7c1-4cc6-b0d9-9db19d562de2"} {"msg":"Generation requested: 106 >= found:106","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:19.521406682Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1563,"":"downstairs","":"downstairs","session_id":"39839b33-e7c1-4cc6-b0d9-9db19d562de2"} {"msg":"Next flush: 168","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:19.521416082Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1563,"":"downstairs","":"downstairs","session_id":"39839b33-e7c1-4cc6-b0d9-9db19d562de2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:19.521425532Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1563,"":"downstairs","":"downstairs","session_id":"39839b33-e7c1-4cc6-b0d9-9db19d562de2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:19.521435352Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1563,"session_id":"39839b33-e7c1-4cc6-b0d9-9db19d562de2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:19.521444172Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1563,"client":"0","":"downstairs","session_id":"39839b33-e7c1-4cc6-b0d9-9db19d562de2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:19.521453882Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1563,"client":"1","":"downstairs","session_id":"39839b33-e7c1-4cc6-b0d9-9db19d562de2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:19.521463652Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1563,"client":"2","":"downstairs","session_id":"39839b33-e7c1-4cc6-b0d9-9db19d562de2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:19.521473532Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1563,"session_id":"39839b33-e7c1-4cc6-b0d9-9db19d562de2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:19.521483892Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1563,"session_id":"39839b33-e7c1-4cc6-b0d9-9db19d562de2"} {"msg":"9b839dc3-a36e-412f-ab00-db9b2e0fb06f is now active with session: 39839b33-e7c1-4cc6-b0d9-9db19d562de2","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:19.521493992Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1563,"session_id":"39839b33-e7c1-4cc6-b0d9-9db19d562de2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:19.521502802Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1563,"session_id":"39839b33-e7c1-4cc6-b0d9-9db19d562de2"} {"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:19.521512022Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1563} {"msg":"Activated sub_volume 9b839dc3-a36e-412f-ab00-db9b2e0fb06f","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:19.521520202Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1563} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:19.521528372Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1563,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:19.521811162Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1563,"task":"crutest"} 01/30 Write block 349 len 8192 data: 9 6 02/30 Write block 573 len 40960 data: 11 11 12 12 12 13 10 7 7 7 03/30 Read block 305 len 32768 04/30 Read block 574 len 16384 05/30 Flush 06/30 Write block 104 len 8192 data: 10 9 07/30 Write block 136 len 36864 data: 7 7 10 13 12 10 7 5 7 08/30 Write block 432 len 20480 data: 5 7 6 5 5 09/30 Write block 178 len 4096 data: 9 10/30 Flush 11/30 Write block 285 len 32768 data: 5 6 6 5 6 7 5 7 12/30 Read block 64 len 28672 13/30 Read block 154 len 8192 14/30 Read block 508 len 40960 15/30 Write block 293 len 8192 data: 10 9 16/30 Flush 17/30 Write block 523 len 24576 data: 7 7 11 10 12 13 18/30 Write block 48 len 16384 data: 6 5 9 10 19/30 Read block 308 len 16384 20/30 Write block 519 len 40960 data: 9 9 9 8 8 8 12 11 13 14 21/30 Write block 184 len 24576 data: 6 8 7 9 9 8 22/30 Read block 95 len 24576 23/30 Read block 238 len 12288 24/30 Flush 25/30 Write block 195 len 8192 data: 7 7 26/30 Write block 460 len 16384 data: 8 7 6 9 27/30 Read block 186 len 16384 28/30 Read block 307 len 20480 29/30 Write block 300 len 36864 data: 11 13 11 10 10 9 9 8 5 30/30 Write block 312 len 40960 data: 6 6 6 5 5 6 7 8 11 10 ---------------------------------------------------------------- Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Sent false 1001 Acked Write 10 Done Done Sent false 1002 Acked Read 8 Done Done Sent false 1003 Acked Read 4 Done Done Sent false 1004 Acked Flush 0 Done Done Sent false 1005 Acked Write 2 Done Done Sent false 1006 Acked Write 9 Done Done Sent false 1007 Acked Write 5 Done Done Sent false 1008 Acked Write 1 Done Done Sent false 1009 Acked Flush 0 Done Done Sent false 1010 Acked Write 8 Done Done Sent false 1011 Acked Read 7 Done Done Sent false 1012 Acked Read 2 Done Done Sent false 1013 Acked Read 10 Done Done Sent false 1014 Acked Write 2 Done Done Sent false 1015 Acked Flush 0 Done Done Sent false 1016 Acked Write 6 Done Done Sent false 1017 Acked Write 4 Done Done Sent false 1018 Acked Read 4 Done Done Sent false 1019 Acked Write 10 Done Done Sent false 1020 Acked Write 6 Done Done Sent false 1021 Acked Read 6 Done Done Sent false 1022 Acked Read 3 Done Done Sent false 1023 Acked Flush 0 Done Done Sent false 1024 Acked Write 2 Done Done Sent false 1025 Acked Write 4 Done Done Sent false 1026 Acked Read 4 Done Done Sent false 1027 Acked Read 5 Done Done Sent false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 30 34 Done 28 28 0 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:23:19.530698466Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1563} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:19.530877566Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1563,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 101 101 101 161 161 161 F F F 1 020-039 103 103 103 163 163 163 F F F 2 040-059 106 106 104 171 171 164 F F F <--- 3 060-079 104 104 104 165 165 165 F F F 4 080-099 104 104 104 165 165 165 F F F 5 100-119 106 106 103 169 169 163 F F F <--- 6 120-139 106 106 104 169 169 165 F F F <--- 7 140-159 106 106 104 169 169 165 F F F <--- 8 160-179 106 106 104 169 169 165 F F F <--- 9 180-199 106 106 98 171 171 153 T T F <--- 10 200-219 100 100 100 160 160 160 F F F 11 220-239 104 104 104 164 164 164 F F F 12 240-259 103 103 103 163 163 163 F F F 13 260-279 100 100 100 160 160 160 F F F 14 280-299 106 106 104 170 170 165 F F F <--- 15 300-319 104 104 104 165 165 165 T T F <--- 16 320-339 103 103 103 163 163 163 T T F <--- 17 340-359 106 106 105 168 168 167 F F F <--- 18 360-379 105 105 105 167 167 167 F F F 19 380-399 103 103 103 163 163 163 F F F 20 400-419 105 105 105 167 167 167 F F F 21 420-439 106 106 102 169 169 162 F F F <--- 22 440-459 105 105 105 167 167 167 F F F 23 460-479 105 105 105 167 167 167 T T F <--- 24 480-499 90 90 90 139 139 139 F F F 25 500-519 106 106 104 171 171 166 F F F <--- 26 520-539 106 106 104 171 171 166 F F F <--- 27 540-559 101 101 101 161 161 161 F F F 28 560-579 106 106 105 168 168 167 F F F <--- 29 580-599 106 106 101 168 168 161 F F F <--- Max gen: 106, Max flush: 171 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 53 Verifying data now No change after verify Loop: 53 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 101 101 101 161 161 161 F F F 1 020-039 103 103 103 163 163 163 F F F 2 040-059 106 106 106 171 171 171 F F F 3 060-079 104 104 104 165 165 165 F F F 4 080-099 104 104 104 165 165 165 F F F 5 100-119 106 106 106 169 169 169 F F F 6 120-139 106 106 106 169 169 169 F F F 7 140-159 106 106 106 169 169 169 F F F 8 160-179 106 106 106 169 169 169 F F F 9 180-199 107 107 107 172 172 172 F F F 10 200-219 100 100 100 160 160 160 F F F 11 220-239 104 104 104 164 164 164 F F F 12 240-259 103 103 103 163 163 163 F F F 13 260-279 100 100 100 160 160 160 F F F 14 280-299 106 106 106 170 170 170 F F F 15 300-319 107 107 107 172 172 172 F F F 16 320-339 107 107 107 172 172 172 F F F 17 340-359 106 106 106 168 168 168 F F F 18 360-379 105 105 105 167 167 167 F F F 19 380-399 103 103 103 163 163 163 F F F 20 400-419 105 105 105 167 167 167 F F F 21 420-439 106 106 106 169 169 169 F F F 22 440-459 105 105 105 167 167 167 F F F 23 460-479 107 107 107 172 172 172 F F F 24 480-499 90 90 90 139 139 139 F F F 25 500-519 106 106 106 171 171 171 F F F 26 520-539 106 106 106 171 171 171 F F F 27 540-559 101 101 101 161 161 161 F F F 28 560-579 106 106 106 168 168 168 F F F 29 580-599 106 106 106 168 168 168 F F F Max gen: 107, Max flush: 172 Begin loop 54 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:23.790182433Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1571,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:23:23.797291384Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1571,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:23.797344894Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1571,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:23.804062444Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1571,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:23.804334825Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1571} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:23.804360755Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1571} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:23.804369335Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1571} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:23.804381215Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1571} {"msg":"Crucible 0f121951-1611-41d8-ab7a-6307c1665029 has session id: 29290511-36ff-4b15-abdc-550f5a0b0035","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:23.804727805Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1571,"session_id":"29290511-36ff-4b15-abdc-550f5a0b0035"} {"msg":"Upstairs opts: Upstairs UUID: 0f121951-1611-41d8-ab7a-6307c1665029, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:23.804746775Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1571,"session_id":"29290511-36ff-4b15-abdc-550f5a0b0035"} {"msg":"Crucible stats registered with UUID: 0f121951-1611-41d8-ab7a-6307c1665029","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:23.804756845Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1571,"session_id":"29290511-36ff-4b15-abdc-550f5a0b0035"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:23:23.804765796Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1571,"session_id":"29290511-36ff-4b15-abdc-550f5a0b0035"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:23.804913126Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1571,"":"io task","client":"2","":"downstairs","session_id":"29290511-36ff-4b15-abdc-550f5a0b0035"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:23.804945246Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1571,"":"io task","client":"0","":"downstairs","session_id":"29290511-36ff-4b15-abdc-550f5a0b0035"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:23.804956076Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1571,"":"io task","client":"1","":"downstairs","session_id":"29290511-36ff-4b15-abdc-550f5a0b0035"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:23.805076046Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1571,"task":"crutest"} {"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:23.805095016Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1571} {"msg":"0f121951-1611-41d8-ab7a-6307c1665029 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:23.805111116Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1571,"session_id":"29290511-36ff-4b15-abdc-550f5a0b0035"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:23.805120036Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1571,"client":"0","":"downstairs","session_id":"29290511-36ff-4b15-abdc-550f5a0b0035"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:23.805129716Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1571,"client":"1","":"downstairs","session_id":"29290511-36ff-4b15-abdc-550f5a0b0035"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:23.805139486Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1571,"client":"2","":"downstairs","session_id":"29290511-36ff-4b15-abdc-550f5a0b0035"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:23.805149126Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1571,"":"io task","client":"2","":"downstairs","session_id":"29290511-36ff-4b15-abdc-550f5a0b0035"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:23.805159346Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1571,"":"io task","client":"0","":"downstairs","session_id":"29290511-36ff-4b15-abdc-550f5a0b0035"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:23.805169406Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1571,"":"io task","client":"1","":"downstairs","session_id":"29290511-36ff-4b15-abdc-550f5a0b0035"} {"msg":"ds_connection connected from Ok(127.0.0.1:44884)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:23.805201556Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1571,"":"io task","client":"0","":"downstairs","session_id":"29290511-36ff-4b15-abdc-550f5a0b0035"} {"msg":"ds_connection connected from Ok(127.0.0.1:63495)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:23.805236666Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1571,"":"io task","client":"2","":"downstairs","session_id":"29290511-36ff-4b15-abdc-550f5a0b0035"} {"msg":"ds_connection connected from Ok(127.0.0.1:53955)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:23.805279726Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1571,"":"io task","client":"1","":"downstairs","session_id":"29290511-36ff-4b15-abdc-550f5a0b0035"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:23.805884427Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1571,"client":"2","":"downstairs","session_id":"29290511-36ff-4b15-abdc-550f5a0b0035"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:23.805904287Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1571,"client":"1","":"downstairs","session_id":"29290511-36ff-4b15-abdc-550f5a0b0035"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:23.805915807Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1571,"client":"0","":"downstairs","session_id":"29290511-36ff-4b15-abdc-550f5a0b0035"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:23.806216868Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1571,"session_id":"29290511-36ff-4b15-abdc-550f5a0b0035"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:23.806226658Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1571,"session_id":"29290511-36ff-4b15-abdc-550f5a0b0035"} {"msg":"[0]R flush_numbers[0..12]: [161, 163, 171, 165, 165, 169, 169, 169, 169, 172, 160, 164]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:23.806265928Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1571,"":"downstairs","":"downstairs","session_id":"29290511-36ff-4b15-abdc-550f5a0b0035"} {"msg":"[0]R generation[0..12]: [101, 103, 106, 104, 104, 106, 106, 106, 106, 107, 100, 104]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:23.806277368Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1571,"":"downstairs","":"downstairs","session_id":"29290511-36ff-4b15-abdc-550f5a0b0035"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:23.806288708Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1571,"":"downstairs","":"downstairs","session_id":"29290511-36ff-4b15-abdc-550f5a0b0035"} {"msg":"[1]R flush_numbers[0..12]: [161, 163, 171, 165, 165, 169, 169, 169, 169, 172, 160, 164]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:23.806299388Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1571,"":"downstairs","":"downstairs","session_id":"29290511-36ff-4b15-abdc-550f5a0b0035"} {"msg":"[1]R generation[0..12]: [101, 103, 106, 104, 104, 106, 106, 106, 106, 107, 100, 104]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:23.806310158Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1571,"":"downstairs","":"downstairs","session_id":"29290511-36ff-4b15-abdc-550f5a0b0035"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:23.806321308Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1571,"":"downstairs","":"downstairs","session_id":"29290511-36ff-4b15-abdc-550f5a0b0035"} {"msg":"[2]R flush_numbers[0..12]: [161, 163, 171, 165, 165, 169, 169, 169, 169, 172, 160, 164]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:23.806334218Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1571,"":"downstairs","":"downstairs","session_id":"29290511-36ff-4b15-abdc-550f5a0b0035"} {"msg":"[2]R generation[0..12]: [101, 103, 106, 104, 104, 106, 106, 106, 106, 107, 100, 104]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:23.806344858Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1571,"":"downstairs","":"downstairs","session_id":"29290511-36ff-4b15-abdc-550f5a0b0035"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:23.806356028Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1571,"":"downstairs","":"downstairs","session_id":"29290511-36ff-4b15-abdc-550f5a0b0035"} {"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:23.806365538Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1571,"":"downstairs","":"downstairs","session_id":"29290511-36ff-4b15-abdc-550f5a0b0035"} {"msg":"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:23.806375418Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1571,"":"downstairs","":"downstairs","session_id":"29290511-36ff-4b15-abdc-550f5a0b0035"} {"msg":"Next flush: 173","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:23.806384988Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1571,"":"downstairs","":"downstairs","session_id":"29290511-36ff-4b15-abdc-550f5a0b0035"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:23.806394538Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1571,"":"downstairs","":"downstairs","session_id":"29290511-36ff-4b15-abdc-550f5a0b0035"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:23.806404368Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1571,"session_id":"29290511-36ff-4b15-abdc-550f5a0b0035"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:23.806413218Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1571,"client":"0","":"downstairs","session_id":"29290511-36ff-4b15-abdc-550f5a0b0035"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:23.806423128Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1571,"client":"1","":"downstairs","session_id":"29290511-36ff-4b15-abdc-550f5a0b0035"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:23.806432858Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1571,"client":"2","":"downstairs","session_id":"29290511-36ff-4b15-abdc-550f5a0b0035"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:23.806442728Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1571,"session_id":"29290511-36ff-4b15-abdc-550f5a0b0035"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:23.806457268Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1571,"session_id":"29290511-36ff-4b15-abdc-550f5a0b0035"} {"msg":"0f121951-1611-41d8-ab7a-6307c1665029 is now active with session: 29290511-36ff-4b15-abdc-550f5a0b0035","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:23.806467468Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1571,"session_id":"29290511-36ff-4b15-abdc-550f5a0b0035"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:23.806476168Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1571,"session_id":"29290511-36ff-4b15-abdc-550f5a0b0035"} {"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:23.806487118Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1571} {"msg":"Activated sub_volume 0f121951-1611-41d8-ab7a-6307c1665029","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:23.806495098Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1571} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:23.806503288Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1571,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:23.806816529Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1571,"task":"crutest"} 01/30 Write block 313 len 28672 data: 7 7 6 6 7 8 9 02/30 Write block 260 len 36864 data: 8 9 10 13 13 14 13 14 15 03/30 Read block 188 len 24576 04/30 Write block 243 len 28672 data: 10 8 10 9 10 10 9 05/30 Read block 564 len 16384 06/30 Read block 452 len 24576 07/30 Read block 500 len 32768 08/30 Write block 413 len 20480 data: 6 7 6 8 7 09/30 Read block 536 len 16384 10/30 Read block 144 len 12288 11/30 Read block 13 len 4096 12/30 Read block 49 len 20480 13/30 Write block 411 len 12288 data: 5 7 7 14/30 Read block 95 len 4096 15/30 Write block 45 len 8192 data: 5 6 16/30 Read block 402 len 4096 17/30 Read block 344 len 40960 18/30 Read block 267 len 20480 19/30 Read block 368 len 32768 20/30 Write block 240 len 16384 data: 12 12 10 11 21/30 Write block 417 len 40960 data: 8 7 6 6 8 8 7 8 7 5 22/30 Write block 517 len 40960 data: 6 7 10 10 10 9 9 9 13 12 23/30 Flush 24/30 Write block 150 len 28672 data: 8 11 10 9 12 11 10 25/30 Read block 162 len 4096 26/30 Read block 342 len 28672 27/30 Read block 573 len 4096 28/30 Read block 165 len 32768 29/30 Write block 571 len 4096 data: 11 30/30 Read block 566 len 24576 ---------------------------------------------------------------- Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Write 9 Sent Done Done false 1002 Acked Read 6 Sent Done Done false 1003 Acked Write 7 Sent Done Done false 1004 Acked Read 4 Sent Done Done false 1005 Acked Read 6 Sent Done Done false 1006 Acked Read 8 Sent Done Done false 1007 Acked Write 5 Sent Done Done false 1008 Acked Read 4 Sent Done Done false 1009 Acked Read 3 Sent Done Done false 1010 Acked Read 1 Sent Done Done false 1011 Acked Read 5 Sent Done Done false 1012 Acked Write 3 Sent Done Done false 1013 Acked Read 1 Sent Done Done false 1014 Acked Write 2 Sent Done Done false 1015 Acked Read 1 Sent Done Done false 1016 Acked Read 10 Sent Done Done false 1017 Acked Read 5 Sent Done Done false 1018 Acked Read 8 Sent Done Done false 1019 Acked Write 4 Sent Done Done false 1020 Acked Write 10 Sent Done Done false 1021 Acked Write 10 Sent Done Done false 1022 Acked Flush 0 Sent Done Done false 1023 Acked Write 7 Sent Done Done false 1024 Acked Read 1 Sent Done Done false 1025 Acked Read 7 Sent Done Done false 1026 Acked Read 1 Sent Done Done false 1027 Acked Read 8 Sent Done Done false 1028 Acked Write 1 Sent Done Done false 1029 Acked Read 6 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 1 0 30 Done 1 29 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:23:23.815757522Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1571} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:23.815944382Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1571,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 101 101 101 161 161 161 F F F 1 020-039 103 103 103 163 163 163 F F F 2 040-059 106 108 108 171 173 173 F F F <--- 3 060-079 104 104 104 165 165 165 F F F 4 080-099 104 104 104 165 165 165 F F F 5 100-119 106 106 106 169 169 169 F F F 6 120-139 106 106 106 169 169 169 F F F 7 140-159 106 106 106 169 169 169 F T T <--- 8 160-179 106 106 106 169 169 169 F F F 9 180-199 107 107 107 172 172 172 F F F 10 200-219 100 100 100 160 160 160 F F F 11 220-239 104 104 104 164 164 164 F F F 12 240-259 103 108 108 163 173 173 F F F <--- 13 260-279 100 108 108 160 173 173 F F F <--- 14 280-299 106 106 106 170 170 170 F F F 15 300-319 107 108 108 172 173 173 T F F <--- 16 320-339 107 107 107 172 172 172 F F F 17 340-359 106 106 106 168 168 168 F F F 18 360-379 105 105 105 167 167 167 F F F 19 380-399 103 103 103 163 163 163 F F F 20 400-419 105 108 108 167 173 173 F F F <--- 21 420-439 106 108 108 169 173 173 F F F <--- 22 440-459 105 105 105 167 167 167 F F F 23 460-479 107 107 107 172 172 172 F F F 24 480-499 90 90 90 139 139 139 F F F 25 500-519 106 108 108 171 173 173 F F F <--- 26 520-539 106 108 108 171 173 173 F F F <--- 27 540-559 101 101 101 161 161 161 F F F 28 560-579 106 106 106 168 168 168 F T T <--- 29 580-599 106 106 106 168 168 168 F F F Max gen: 108, Max flush: 173 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 54 Verifying data now No change after verify Loop: 54 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 101 101 101 161 161 161 F F F 1 020-039 103 103 103 163 163 163 F F F 2 040-059 108 108 108 173 173 173 F F F 3 060-079 104 104 104 165 165 165 F F F 4 080-099 104 104 104 165 165 165 F F F 5 100-119 106 106 106 169 169 169 F F F 6 120-139 106 106 106 169 169 169 F F F 7 140-159 109 109 109 174 174 174 F F F 8 160-179 106 106 106 169 169 169 F F F 9 180-199 107 107 107 172 172 172 F F F 10 200-219 100 100 100 160 160 160 F F F 11 220-239 104 104 104 164 164 164 F F F 12 240-259 108 108 108 173 173 173 F F F 13 260-279 108 108 108 173 173 173 F F F 14 280-299 106 106 106 170 170 170 F F F 15 300-319 108 108 108 173 173 173 F F F 16 320-339 107 107 107 172 172 172 F F F 17 340-359 106 106 106 168 168 168 F F F 18 360-379 105 105 105 167 167 167 F F F 19 380-399 103 103 103 163 163 163 F F F 20 400-419 108 108 108 173 173 173 F F F 21 420-439 108 108 108 173 173 173 F F F 22 440-459 105 105 105 167 167 167 F F F 23 460-479 107 107 107 172 172 172 F F F 24 480-499 90 90 90 139 139 139 F F F 25 500-519 108 108 108 173 173 173 F F F 26 520-539 108 108 108 173 173 173 F F F 27 540-559 101 101 101 161 161 161 F F F 28 560-579 109 109 109 174 174 174 F F F 29 580-599 106 106 106 168 168 168 F F F Max gen: 109, Max flush: 174 Begin loop 55 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:28.065592835Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1579,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:28.073867847Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1579,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:28.074159478Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1579} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:28.074188958Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1579} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:28.074197648Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1579} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:28.074209708Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1579} {"msg":"Crucible 4f7d358f-54e7-40d7-96be-ef20f9357a38 has session id: 30d66e1e-8fbf-4b80-bd13-dc46fc6328b7","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:28.074562078Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1579,"session_id":"30d66e1e-8fbf-4b80-bd13-dc46fc6328b7"} {"msg":"Upstairs opts: Upstairs UUID: 4f7d358f-54e7-40d7-96be-ef20f9357a38, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:28.074587078Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1579,"session_id":"30d66e1e-8fbf-4b80-bd13-dc46fc6328b7"} {"msg":"Crucible stats registered with UUID: 4f7d358f-54e7-40d7-96be-ef20f9357a38","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:28.074597358Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1579,"session_id":"30d66e1e-8fbf-4b80-bd13-dc46fc6328b7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:23:28.074606498Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1579,"session_id":"30d66e1e-8fbf-4b80-bd13-dc46fc6328b7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:28.074753839Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1579,"":"io task","client":"0","":"downstairs","session_id":"30d66e1e-8fbf-4b80-bd13-dc46fc6328b7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:28.074787089Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1579,"":"io task","client":"2","":"downstairs","session_id":"30d66e1e-8fbf-4b80-bd13-dc46fc6328b7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:28.074798109Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1579,"":"io task","client":"1","":"downstairs","session_id":"30d66e1e-8fbf-4b80-bd13-dc46fc6328b7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:28.074944419Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1579,"task":"crutest"} {"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:28.074960269Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1579} {"msg":"4f7d358f-54e7-40d7-96be-ef20f9357a38 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:28.075006089Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1579,"session_id":"30d66e1e-8fbf-4b80-bd13-dc46fc6328b7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:28.075015609Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1579,"client":"0","":"downstairs","session_id":"30d66e1e-8fbf-4b80-bd13-dc46fc6328b7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:28.075025919Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1579,"client":"1","":"downstairs","session_id":"30d66e1e-8fbf-4b80-bd13-dc46fc6328b7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:28.075035729Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1579,"client":"2","":"downstairs","session_id":"30d66e1e-8fbf-4b80-bd13-dc46fc6328b7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:28.075045489Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1579,"":"io task","client":"2","":"downstairs","session_id":"30d66e1e-8fbf-4b80-bd13-dc46fc6328b7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:28.075081919Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1579,"":"io task","client":"0","":"downstairs","session_id":"30d66e1e-8fbf-4b80-bd13-dc46fc6328b7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:28.075105049Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1579,"":"io task","client":"1","":"downstairs","session_id":"30d66e1e-8fbf-4b80-bd13-dc46fc6328b7"} {"msg":"ds_connection connected from Ok(127.0.0.1:52043)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:28.075116359Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1579,"":"io task","client":"2","":"downstairs","session_id":"30d66e1e-8fbf-4b80-bd13-dc46fc6328b7"} {"msg":"ds_connection connected from Ok(127.0.0.1:51174)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:28.075131189Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1579,"":"io task","client":"1","":"downstairs","session_id":"30d66e1e-8fbf-4b80-bd13-dc46fc6328b7"} {"msg":"ds_connection connected from Ok(127.0.0.1:46700)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:28.075237799Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1579,"":"io task","client":"0","":"downstairs","session_id":"30d66e1e-8fbf-4b80-bd13-dc46fc6328b7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:28.07575018Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1579,"client":"0","":"downstairs","session_id":"30d66e1e-8fbf-4b80-bd13-dc46fc6328b7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:28.07579592Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1579,"client":"2","":"downstairs","session_id":"30d66e1e-8fbf-4b80-bd13-dc46fc6328b7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:28.07585079Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1579,"client":"1","":"downstairs","session_id":"30d66e1e-8fbf-4b80-bd13-dc46fc6328b7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:28.076104681Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1579,"session_id":"30d66e1e-8fbf-4b80-bd13-dc46fc6328b7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:28.076151221Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1579,"session_id":"30d66e1e-8fbf-4b80-bd13-dc46fc6328b7"} {"msg":"[0]R flush_numbers[0..12]: [161, 163, 173, 165, 165, 169, 169, 174, 169, 172, 160, 164]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:28.076217061Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1579,"":"downstairs","":"downstairs","session_id":"30d66e1e-8fbf-4b80-bd13-dc46fc6328b7"} {"msg":"[0]R generation[0..12]: [101, 103, 108, 104, 104, 106, 106, 109, 106, 107, 100, 104]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:28.076228401Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1579,"":"downstairs","":"downstairs","session_id":"30d66e1e-8fbf-4b80-bd13-dc46fc6328b7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:28.076239561Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1579,"":"downstairs","":"downstairs","session_id":"30d66e1e-8fbf-4b80-bd13-dc46fc6328b7"} {"msg":"[1]R flush_numbers[0..12]: [161, 163, 173, 165, 165, 169, 169, 174, 169, 172, 160, 164]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:28.076250441Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1579,"":"downstairs","":"downstairs","session_id":"30d66e1e-8fbf-4b80-bd13-dc46fc6328b7"} {"msg":"[1]R generation[0..12]: [101, 103, 108, 104, 104, 106, 106, 109, 106, 107, 100, 104]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:28.076261211Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1579,"":"downstairs","":"downstairs","session_id":"30d66e1e-8fbf-4b80-bd13-dc46fc6328b7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:28.076272271Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1579,"":"downstairs","":"downstairs","session_id":"30d66e1e-8fbf-4b80-bd13-dc46fc6328b7"} {"msg":"[2]R flush_numbers[0..12]: [161, 163, 173, 165, 165, 169, 169, 174, 169, 172, 160, 164]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:28.076283081Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1579,"":"downstairs","":"downstairs","session_id":"30d66e1e-8fbf-4b80-bd13-dc46fc6328b7"} {"msg":"[2]R generation[0..12]: [101, 103, 108, 104, 104, 106, 106, 109, 106, 107, 100, 104]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:28.076293701Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1579,"":"downstairs","":"downstairs","session_id":"30d66e1e-8fbf-4b80-bd13-dc46fc6328b7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:28.076304661Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1579,"":"downstairs","":"downstairs","session_id":"30d66e1e-8fbf-4b80-bd13-dc46fc6328b7"} {"msg":"Max found gen is 110","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:28.076314221Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1579,"":"downstairs","":"downstairs","session_id":"30d66e1e-8fbf-4b80-bd13-dc46fc6328b7"} {"msg":"Generation requested: 110 >= found:110","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:28.076324161Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1579,"":"downstairs","":"downstairs","session_id":"30d66e1e-8fbf-4b80-bd13-dc46fc6328b7"} {"msg":"Next flush: 175","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:28.076335881Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1579,"":"downstairs","":"downstairs","session_id":"30d66e1e-8fbf-4b80-bd13-dc46fc6328b7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:28.076345351Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1579,"":"downstairs","":"downstairs","session_id":"30d66e1e-8fbf-4b80-bd13-dc46fc6328b7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:28.076355431Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1579,"session_id":"30d66e1e-8fbf-4b80-bd13-dc46fc6328b7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:28.076364331Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1579,"client":"0","":"downstairs","session_id":"30d66e1e-8fbf-4b80-bd13-dc46fc6328b7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:28.076374481Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1579,"client":"1","":"downstairs","session_id":"30d66e1e-8fbf-4b80-bd13-dc46fc6328b7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:28.076384321Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1579,"client":"2","":"downstairs","session_id":"30d66e1e-8fbf-4b80-bd13-dc46fc6328b7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:28.076394521Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1579,"session_id":"30d66e1e-8fbf-4b80-bd13-dc46fc6328b7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:28.076403901Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1579,"session_id":"30d66e1e-8fbf-4b80-bd13-dc46fc6328b7"} {"msg":"4f7d358f-54e7-40d7-96be-ef20f9357a38 is now active with session: 30d66e1e-8fbf-4b80-bd13-dc46fc6328b7","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:28.076414021Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1579,"session_id":"30d66e1e-8fbf-4b80-bd13-dc46fc6328b7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:28.076422871Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1579,"session_id":"30d66e1e-8fbf-4b80-bd13-dc46fc6328b7"} {"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:28.076432011Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1579} {"msg":"Activated sub_volume 4f7d358f-54e7-40d7-96be-ef20f9357a38","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:28.076440261Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1579} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:28.076448351Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1579,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:28.076734122Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1579,"task":"crutest"} 01/30 Write block 168 len 8192 data: 8 8 02/30 Write block 465 len 28672 data: 5 6 8 10 9 8 8 03/30 Read block 469 len 28672 04/30 Read block 316 len 20480 05/30 Read block 493 len 20480 06/30 Write block 125 len 32768 data: 8 7 5 6 5 5 6 7 07/30 Write block 70 len 20480 data: 10 8 8 8 7 08/30 Write block 250 len 36864 data: 8 10 8 7 7 8 6 6 8 09/30 Read block 75 len 12288 10/30 Read block 254 len 40960 11/30 Write block 156 len 16384 data: 11 9 12 10 12/30 Flush 13/30 Write block 167 len 36864 data: 7 9 9 8 8 9 9 7 5 14/30 Write block 387 len 20480 data: 8 10 10 8 8 15/30 Flush 16/30 Write block 52 len 16384 data: 11 10 9 10 17/30 Write block 88 len 12288 data: 10 11 10 18/30 Flush 19/30 Write block 228 len 32768 data: 8 5 5 4 5 6 7 7 20/30 Read block 535 len 20480 21/30 Read block 26 len 4096 22/30 Write block 9 len 36864 data: 8 8 7 9 7 8 7 7 8 23/30 Flush 24/30 Write block 511 len 20480 data: 7 9 8 7 5 25/30 Write block 112 len 24576 data: 8 9 9 9 9 7 26/30 Write block 432 len 32768 data: 6 8 7 6 6 8 8 8 27/30 Write block 513 len 4096 data: 9 28/30 Read block 461 len 4096 29/30 Read block 359 len 16384 30/30 Write block 376 len 28672 data: 13 11 11 13 12 12 9 ---------------------------------------------------------------- Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Write 7 Done Done Done false 1002 Acked Read 7 Done Done Done false 1003 Acked Read 5 Done Done Done false 1004 Acked Read 5 Done Done Done false 1005 Acked Write 8 Done Done Done false 1006 Acked Write 5 Done Sent Done false 1007 Acked Write 9 Done Sent Done false 1008 Acked Read 3 Done Sent Done false 1009 Acked Read 10 Done Sent Done false 1010 Acked Write 4 Done Sent Done false 1011 Acked Flush 0 Done Sent Done false 1012 Acked Write 9 Done Sent Done false 1013 Acked Write 5 Done Sent Done false 1014 Acked Flush 0 Done Sent Done false 1015 Acked Write 4 Done Sent Done false 1016 Acked Write 3 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Write 8 Done Sent Done false 1019 Acked Read 5 Done Sent Done false 1020 Acked Read 1 Done Sent Done false 1021 Acked Write 9 Done Sent Done false 1022 Acked Flush 0 Done Sent Done false 1023 Acked Write 5 Done Sent Done false 1024 Acked Write 6 Done Sent Done false 1025 Acked Write 8 Done Sent Done false 1026 Acked Write 1 Done Sent Done false 1027 Acked Read 1 Done Sent Done false 1028 Acked Read 4 Done Sent Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 24 1 26 Done 29 6 29 64 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) None Some(JobId(1022)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:23:28.085118944Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1579} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:28.085284784Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1579,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 110 101 110 178 161 178 F F F <--- 1 020-039 103 103 103 163 163 163 F F F 2 040-059 110 108 110 177 173 177 F F F <--- 3 060-079 110 104 110 175 165 175 F F F <--- 4 080-099 110 104 110 177 165 177 F F F <--- 5 100-119 106 106 106 169 169 169 T F T <--- 6 120-139 110 106 110 175 169 175 F T F <--- 7 140-159 110 109 110 175 174 175 F F F <--- 8 160-179 110 106 110 176 169 176 F T F <--- 9 180-199 107 107 107 172 172 172 F F F 10 200-219 100 100 100 160 160 160 F F F 11 220-239 110 104 110 178 164 178 F F F <--- 12 240-259 110 108 110 175 173 175 F F F <--- 13 260-279 108 108 108 173 173 173 F F F 14 280-299 106 106 106 170 170 170 F F F 15 300-319 108 108 108 173 173 173 F F F 16 320-339 107 107 107 172 172 172 F F F 17 340-359 106 106 106 168 168 168 F F F 18 360-379 105 105 105 167 167 167 T F T <--- 19 380-399 110 103 110 176 163 176 T F T <--- 20 400-419 108 108 108 173 173 173 F F F 21 420-439 108 108 108 173 173 173 T F T <--- 22 440-459 105 105 105 167 167 167 F F F 23 460-479 110 107 110 175 172 175 F T F <--- 24 480-499 90 90 90 139 139 139 F F F 25 500-519 108 108 108 173 173 173 T F T <--- 26 520-539 108 108 108 173 173 173 F F F 27 540-559 101 101 101 161 161 161 F F F 28 560-579 109 109 109 174 174 174 F F F 29 580-599 106 106 106 168 168 168 F F F Max gen: 110, Max flush: 178 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 55 Verifying data now No change after verify Loop: 55 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 110 110 110 178 178 178 F F F 1 020-039 103 103 103 163 163 163 F F F 2 040-059 110 110 110 177 177 177 F F F 3 060-079 110 110 110 175 175 175 F F F 4 080-099 110 110 110 177 177 177 F F F 5 100-119 111 111 111 179 179 179 F F F 6 120-139 110 110 110 175 175 175 F F F 7 140-159 110 110 110 175 175 175 F F F 8 160-179 110 110 110 176 176 176 F F F 9 180-199 107 107 107 172 172 172 F F F 10 200-219 100 100 100 160 160 160 F F F 11 220-239 110 110 110 178 178 178 F F F 12 240-259 110 110 110 175 175 175 F F F 13 260-279 108 108 108 173 173 173 F F F 14 280-299 106 106 106 170 170 170 F F F 15 300-319 108 108 108 173 173 173 F F F 16 320-339 107 107 107 172 172 172 F F F 17 340-359 106 106 106 168 168 168 F F F 18 360-379 111 111 111 179 179 179 F F F 19 380-399 111 111 111 179 179 179 F F F 20 400-419 108 108 108 173 173 173 F F F 21 420-439 111 111 111 179 179 179 F F F 22 440-459 105 105 105 167 167 167 F F F 23 460-479 110 110 110 175 175 175 F F F 24 480-499 90 90 90 139 139 139 F F F 25 500-519 111 111 111 179 179 179 F F F 26 520-539 108 108 108 173 173 173 F F F 27 540-559 101 101 101 161 161 161 F F F 28 560-579 109 109 109 174 174 174 F F F 29 580-599 106 106 106 168 168 168 F F F Max gen: 111, Max flush: 179 Begin loop 56 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:32.335835209Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1587,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:23:32.342968049Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1587,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:32.34302496Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1587,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:32.3500052Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1587,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:32.35028312Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1587} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:32.350311361Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1587} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:32.350319881Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1587} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:32.350331831Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1587} {"msg":"Crucible 09639e85-1557-4769-9d00-a7aac4d5d989 has session id: 5eddf939-59ce-4dba-a3d7-06454143c09d","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:32.350661761Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1587,"session_id":"5eddf939-59ce-4dba-a3d7-06454143c09d"} {"msg":"Upstairs opts: Upstairs UUID: 09639e85-1557-4769-9d00-a7aac4d5d989, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:32.350692461Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1587,"session_id":"5eddf939-59ce-4dba-a3d7-06454143c09d"} {"msg":"Crucible stats registered with UUID: 09639e85-1557-4769-9d00-a7aac4d5d989","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:32.350703521Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1587,"session_id":"5eddf939-59ce-4dba-a3d7-06454143c09d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:23:32.350712951Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1587,"session_id":"5eddf939-59ce-4dba-a3d7-06454143c09d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:32.350844401Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1587,"":"io task","client":"1","":"downstairs","session_id":"5eddf939-59ce-4dba-a3d7-06454143c09d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:32.350876791Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1587,"":"io task","client":"0","":"downstairs","session_id":"5eddf939-59ce-4dba-a3d7-06454143c09d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:32.350888101Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1587,"":"io task","client":"2","":"downstairs","session_id":"5eddf939-59ce-4dba-a3d7-06454143c09d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:32.351031942Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1587,"task":"crutest"} {"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:32.351046622Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1587} {"msg":"09639e85-1557-4769-9d00-a7aac4d5d989 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:32.351081372Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1587,"session_id":"5eddf939-59ce-4dba-a3d7-06454143c09d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:32.351090842Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1587,"client":"0","":"downstairs","session_id":"5eddf939-59ce-4dba-a3d7-06454143c09d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:32.351100572Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1587,"client":"1","":"downstairs","session_id":"5eddf939-59ce-4dba-a3d7-06454143c09d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:32.351110372Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1587,"client":"2","":"downstairs","session_id":"5eddf939-59ce-4dba-a3d7-06454143c09d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:32.351120122Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1587,"":"io task","client":"2","":"downstairs","session_id":"5eddf939-59ce-4dba-a3d7-06454143c09d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:32.351130402Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1587,"":"io task","client":"0","":"downstairs","session_id":"5eddf939-59ce-4dba-a3d7-06454143c09d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:32.351140452Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1587,"":"io task","client":"1","":"downstairs","session_id":"5eddf939-59ce-4dba-a3d7-06454143c09d"} {"msg":"ds_connection connected from Ok(127.0.0.1:36606)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:32.351205672Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1587,"":"io task","client":"1","":"downstairs","session_id":"5eddf939-59ce-4dba-a3d7-06454143c09d"} {"msg":"ds_connection connected from Ok(127.0.0.1:60148)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:32.351216592Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1587,"":"io task","client":"0","":"downstairs","session_id":"5eddf939-59ce-4dba-a3d7-06454143c09d"} {"msg":"ds_connection connected from Ok(127.0.0.1:46115)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:32.351227062Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1587,"":"io task","client":"2","":"downstairs","session_id":"5eddf939-59ce-4dba-a3d7-06454143c09d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:32.351917903Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1587,"client":"2","":"downstairs","session_id":"5eddf939-59ce-4dba-a3d7-06454143c09d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:32.351936663Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1587,"client":"1","":"downstairs","session_id":"5eddf939-59ce-4dba-a3d7-06454143c09d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:32.351972753Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1587,"client":"0","":"downstairs","session_id":"5eddf939-59ce-4dba-a3d7-06454143c09d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:32.352254753Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1587,"session_id":"5eddf939-59ce-4dba-a3d7-06454143c09d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:32.352265493Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1587,"session_id":"5eddf939-59ce-4dba-a3d7-06454143c09d"} {"msg":"[0]R flush_numbers[0..12]: [178, 163, 177, 175, 177, 179, 175, 175, 176, 172, 160, 178]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:32.352308594Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1587,"":"downstairs","":"downstairs","session_id":"5eddf939-59ce-4dba-a3d7-06454143c09d"} {"msg":"[0]R generation[0..12]: [110, 103, 110, 110, 110, 111, 110, 110, 110, 107, 100, 110]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:32.352320144Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1587,"":"downstairs","":"downstairs","session_id":"5eddf939-59ce-4dba-a3d7-06454143c09d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:32.352331394Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1587,"":"downstairs","":"downstairs","session_id":"5eddf939-59ce-4dba-a3d7-06454143c09d"} {"msg":"[1]R flush_numbers[0..12]: [178, 163, 177, 175, 177, 179, 175, 175, 176, 172, 160, 178]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:32.352342124Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1587,"":"downstairs","":"downstairs","session_id":"5eddf939-59ce-4dba-a3d7-06454143c09d"} {"msg":"[1]R generation[0..12]: [110, 103, 110, 110, 110, 111, 110, 110, 110, 107, 100, 110]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:32.352352924Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1587,"":"downstairs","":"downstairs","session_id":"5eddf939-59ce-4dba-a3d7-06454143c09d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:32.352364144Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1587,"":"downstairs","":"downstairs","session_id":"5eddf939-59ce-4dba-a3d7-06454143c09d"} {"msg":"[2]R flush_numbers[0..12]: [178, 163, 177, 175, 177, 179, 175, 175, 176, 172, 160, 178]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:32.352374924Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1587,"":"downstairs","":"downstairs","session_id":"5eddf939-59ce-4dba-a3d7-06454143c09d"} {"msg":"[2]R generation[0..12]: [110, 103, 110, 110, 110, 111, 110, 110, 110, 107, 100, 110]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:32.352385724Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1587,"":"downstairs","":"downstairs","session_id":"5eddf939-59ce-4dba-a3d7-06454143c09d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:32.352396714Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1587,"":"downstairs","":"downstairs","session_id":"5eddf939-59ce-4dba-a3d7-06454143c09d"} {"msg":"Max found gen is 112","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:32.352406174Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1587,"":"downstairs","":"downstairs","session_id":"5eddf939-59ce-4dba-a3d7-06454143c09d"} {"msg":"Generation requested: 112 >= found:112","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:32.352416084Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1587,"":"downstairs","":"downstairs","session_id":"5eddf939-59ce-4dba-a3d7-06454143c09d"} {"msg":"Next flush: 180","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:32.352425594Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1587,"":"downstairs","":"downstairs","session_id":"5eddf939-59ce-4dba-a3d7-06454143c09d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:32.352435084Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1587,"":"downstairs","":"downstairs","session_id":"5eddf939-59ce-4dba-a3d7-06454143c09d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:32.352444904Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1587,"session_id":"5eddf939-59ce-4dba-a3d7-06454143c09d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:32.352455334Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1587,"client":"0","":"downstairs","session_id":"5eddf939-59ce-4dba-a3d7-06454143c09d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:32.352465014Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1587,"client":"1","":"downstairs","session_id":"5eddf939-59ce-4dba-a3d7-06454143c09d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:32.352474724Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1587,"client":"2","":"downstairs","session_id":"5eddf939-59ce-4dba-a3d7-06454143c09d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:32.352484614Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1587,"session_id":"5eddf939-59ce-4dba-a3d7-06454143c09d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:32.352493914Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1587,"session_id":"5eddf939-59ce-4dba-a3d7-06454143c09d"} {"msg":"09639e85-1557-4769-9d00-a7aac4d5d989 is now active with session: 5eddf939-59ce-4dba-a3d7-06454143c09d","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:32.352504094Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1587,"session_id":"5eddf939-59ce-4dba-a3d7-06454143c09d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:32.352512994Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1587,"session_id":"5eddf939-59ce-4dba-a3d7-06454143c09d"} {"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:32.352522134Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1587} {"msg":"Activated sub_volume 09639e85-1557-4769-9d00-a7aac4d5d989","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:32.352530224Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1587} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:32.352538554Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1587,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:32.352867374Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1587,"task":"crutest"} 01/30 Write block 465 len 28672 data: 6 7 9 11 10 9 9 02/30 Read block 548 len 8192 03/30 Read block 2 len 40960 04/30 Write block 487 len 4096 data: 12 05/30 Read block 415 len 24576 06/30 Write block 55 len 8192 data: 11 8 07/30 Read block 175 len 8192 08/30 Read block 57 len 4096 09/30 Read block 67 len 8192 10/30 Write block 486 len 28672 data: 11 13 10 11 11 8 10 11/30 Read block 553 len 28672 12/30 Write block 513 len 36864 data: 10 8 6 5 7 8 11 11 11 13/30 Write block 208 len 8192 data: 9 10 14/30 Write block 536 len 8192 data: 7 8 15/30 Write block 482 len 4096 data: 5 16/30 Read block 114 len 8192 17/30 Read block 257 len 40960 18/30 Read block 364 len 20480 19/30 Read block 457 len 32768 20/30 Read block 431 len 40960 21/30 Write block 197 len 12288 data: 8 7 6 22/30 Flush 23/30 Write block 166 len 24576 data: 5 8 10 10 9 9 24/30 Read block 574 len 20480 25/30 Read block 376 len 12288 26/30 Write block 410 len 32768 data: 8 6 8 8 8 7 9 9 27/30 Write block 38 len 24576 data: 7 8 6 7 7 6 28/30 Read block 401 len 20480 29/30 Write block 193 len 20480 data: 11 7 8 8 9 30/30 Read block 436 len 20480 ---------------------------------------------------------------- Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Read 2 Done Done Done false 1002 Acked Read 10 Done Done Done false 1003 Acked Write 1 Done Done Done false 1004 Acked Read 6 Sent Done Done false 1005 Acked Write 2 Sent Done Done false 1006 Acked Read 2 Sent Done Done false 1007 Acked Read 1 Sent Done Done false 1008 Acked Read 2 Sent Done Done false 1009 Acked Write 7 Sent Done Done false 1010 Acked Read 7 Sent Done Done false 1011 Acked Write 9 Sent Done Done false 1012 Acked Write 2 Sent Done Done false 1013 Acked Write 2 Sent Done Done false 1014 Acked Write 1 Sent Done Done false 1015 Acked Read 2 Sent Done Done false 1016 Acked Read 10 Sent Done Done false 1017 Acked Read 5 Sent Done Done false 1018 Acked Read 8 Sent Done Done false 1019 Acked Read 10 Sent Done Done false 1020 Acked Write 3 Sent Done Done false 1021 Acked Flush 0 Sent Done Done false 1022 Acked Write 6 Sent Done Done false 1023 Acked Read 5 Sent Done Done false 1024 Acked Read 3 Sent Done Done false 1025 Acked Write 8 Sent Done Done false 1026 Acked Write 6 Sent Done Done false 1027 Acked Read 5 Sent Done Done false 1028 Acked Write 5 Sent Done Done false 1029 Acked Read 5 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 26 0 0 26 Done 4 30 30 64 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:23:32.361568278Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1587} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:32.361749108Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1587,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 110 110 110 178 178 178 F F F 1 020-039 103 103 103 163 163 163 F T T <--- 2 040-059 110 112 112 177 180 180 F T T <--- 3 060-079 110 110 110 175 175 175 F F F 4 080-099 110 110 110 177 177 177 F F F 5 100-119 111 111 111 179 179 179 F F F 6 120-139 110 110 110 175 175 175 F F F 7 140-159 110 110 110 175 175 175 F F F 8 160-179 110 110 110 176 176 176 F T T <--- 9 180-199 107 112 112 172 180 180 F T T <--- 10 200-219 100 112 112 160 180 180 F F F <--- 11 220-239 110 110 110 178 178 178 F F F 12 240-259 110 110 110 175 175 175 F F F 13 260-279 108 108 108 173 173 173 F F F 14 280-299 106 106 106 170 170 170 F F F 15 300-319 108 108 108 173 173 173 F F F 16 320-339 107 107 107 172 172 172 F F F 17 340-359 106 106 106 168 168 168 F F F 18 360-379 111 111 111 179 179 179 F F F 19 380-399 111 111 111 179 179 179 F F F 20 400-419 108 108 108 173 173 173 F T T <--- 21 420-439 111 111 111 179 179 179 F F F 22 440-459 105 105 105 167 167 167 F F F 23 460-479 110 112 112 175 180 180 T F F <--- 24 480-499 90 112 112 139 180 180 T F F <--- 25 500-519 111 112 112 179 180 180 F F F <--- 26 520-539 108 112 112 173 180 180 F F F <--- 27 540-559 101 101 101 161 161 161 F F F 28 560-579 109 109 109 174 174 174 F F F 29 580-599 106 106 106 168 168 168 F F F Max gen: 112, Max flush: 180 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 56 Verifying data now No change after verify Loop: 56 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 110 110 110 178 178 178 F F F 1 020-039 113 113 113 181 181 181 F F F 2 040-059 113 113 113 181 181 181 F F F 3 060-079 110 110 110 175 175 175 F F F 4 080-099 110 110 110 177 177 177 F F F 5 100-119 111 111 111 179 179 179 F F F 6 120-139 110 110 110 175 175 175 F F F 7 140-159 110 110 110 175 175 175 F F F 8 160-179 113 113 113 181 181 181 F F F 9 180-199 113 113 113 181 181 181 F F F 10 200-219 112 112 112 180 180 180 F F F 11 220-239 110 110 110 178 178 178 F F F 12 240-259 110 110 110 175 175 175 F F F 13 260-279 108 108 108 173 173 173 F F F 14 280-299 106 106 106 170 170 170 F F F 15 300-319 108 108 108 173 173 173 F F F 16 320-339 107 107 107 172 172 172 F F F 17 340-359 106 106 106 168 168 168 F F F 18 360-379 111 111 111 179 179 179 F F F 19 380-399 111 111 111 179 179 179 F F F 20 400-419 113 113 113 181 181 181 F F F 21 420-439 111 111 111 179 179 179 F F F 22 440-459 105 105 105 167 167 167 F F F 23 460-479 112 112 112 180 180 180 F F F 24 480-499 112 112 112 180 180 180 F F F 25 500-519 112 112 112 180 180 180 F F F 26 520-539 112 112 112 180 180 180 F F F 27 540-559 101 101 101 161 161 161 F F F 28 560-579 109 109 109 174 174 174 F F F 29 580-599 106 106 106 168 168 168 F F F Max gen: 113, Max flush: 181 Begin loop 57 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:36.61739611Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1595,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:36.625603982Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1595,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:36.625892862Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1595} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:36.625920423Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1595} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:36.625928943Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1595} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:36.625942813Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1595} {"msg":"Crucible 12cf73ec-6903-4e28-9f40-3aa52283d92d has session id: ca10a56b-ee54-43e4-bcfe-cef963f7f0b3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:36.626248433Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1595,"session_id":"ca10a56b-ee54-43e4-bcfe-cef963f7f0b3"} {"msg":"Upstairs opts: Upstairs UUID: 12cf73ec-6903-4e28-9f40-3aa52283d92d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:36.626278183Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1595,"session_id":"ca10a56b-ee54-43e4-bcfe-cef963f7f0b3"} {"msg":"Crucible stats registered with UUID: 12cf73ec-6903-4e28-9f40-3aa52283d92d","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:36.626289233Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1595,"session_id":"ca10a56b-ee54-43e4-bcfe-cef963f7f0b3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:23:36.626298563Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1595,"session_id":"ca10a56b-ee54-43e4-bcfe-cef963f7f0b3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:36.626436913Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1595,"":"io task","client":"2","":"downstairs","session_id":"ca10a56b-ee54-43e4-bcfe-cef963f7f0b3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:36.626454723Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1595,"":"io task","client":"0","":"downstairs","session_id":"ca10a56b-ee54-43e4-bcfe-cef963f7f0b3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:36.626464933Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1595,"":"io task","client":"1","":"downstairs","session_id":"ca10a56b-ee54-43e4-bcfe-cef963f7f0b3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:36.626636704Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1595,"task":"crutest"} {"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:36.626652364Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1595} {"msg":"12cf73ec-6903-4e28-9f40-3aa52283d92d active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:36.626669944Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1595,"session_id":"ca10a56b-ee54-43e4-bcfe-cef963f7f0b3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:36.626679024Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1595,"client":"0","":"downstairs","session_id":"ca10a56b-ee54-43e4-bcfe-cef963f7f0b3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:36.626688894Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1595,"client":"1","":"downstairs","session_id":"ca10a56b-ee54-43e4-bcfe-cef963f7f0b3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:36.626698714Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1595,"client":"2","":"downstairs","session_id":"ca10a56b-ee54-43e4-bcfe-cef963f7f0b3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:36.626708364Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1595,"":"io task","client":"2","":"downstairs","session_id":"ca10a56b-ee54-43e4-bcfe-cef963f7f0b3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:36.626720944Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1595,"":"io task","client":"0","":"downstairs","session_id":"ca10a56b-ee54-43e4-bcfe-cef963f7f0b3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:36.626731074Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1595,"":"io task","client":"1","":"downstairs","session_id":"ca10a56b-ee54-43e4-bcfe-cef963f7f0b3"} {"msg":"ds_connection connected from Ok(127.0.0.1:45785)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:36.626803334Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1595,"":"io task","client":"1","":"downstairs","session_id":"ca10a56b-ee54-43e4-bcfe-cef963f7f0b3"} {"msg":"ds_connection connected from Ok(127.0.0.1:62737)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:36.626828634Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1595,"":"io task","client":"0","":"downstairs","session_id":"ca10a56b-ee54-43e4-bcfe-cef963f7f0b3"} {"msg":"ds_connection connected from Ok(127.0.0.1:42455)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:36.626840044Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1595,"":"io task","client":"2","":"downstairs","session_id":"ca10a56b-ee54-43e4-bcfe-cef963f7f0b3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:36.627552155Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1595,"client":"0","":"downstairs","session_id":"ca10a56b-ee54-43e4-bcfe-cef963f7f0b3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:36.627578115Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1595,"client":"1","":"downstairs","session_id":"ca10a56b-ee54-43e4-bcfe-cef963f7f0b3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:36.627590165Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1595,"client":"2","":"downstairs","session_id":"ca10a56b-ee54-43e4-bcfe-cef963f7f0b3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:36.627871025Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1595,"session_id":"ca10a56b-ee54-43e4-bcfe-cef963f7f0b3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:36.627880715Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1595,"session_id":"ca10a56b-ee54-43e4-bcfe-cef963f7f0b3"} {"msg":"[0]R flush_numbers[0..12]: [178, 181, 181, 175, 177, 179, 175, 175, 181, 181, 180, 178]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:36.627921426Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1595,"":"downstairs","":"downstairs","session_id":"ca10a56b-ee54-43e4-bcfe-cef963f7f0b3"} {"msg":"[0]R generation[0..12]: [110, 113, 113, 110, 110, 111, 110, 110, 113, 113, 112, 110]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:36.627932716Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1595,"":"downstairs","":"downstairs","session_id":"ca10a56b-ee54-43e4-bcfe-cef963f7f0b3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:36.627943926Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1595,"":"downstairs","":"downstairs","session_id":"ca10a56b-ee54-43e4-bcfe-cef963f7f0b3"} {"msg":"[1]R flush_numbers[0..12]: [178, 181, 181, 175, 177, 179, 175, 175, 181, 181, 180, 178]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:36.627954816Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1595,"":"downstairs","":"downstairs","session_id":"ca10a56b-ee54-43e4-bcfe-cef963f7f0b3"} {"msg":"[1]R generation[0..12]: [110, 113, 113, 110, 110, 111, 110, 110, 113, 113, 112, 110]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:36.627967656Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1595,"":"downstairs","":"downstairs","session_id":"ca10a56b-ee54-43e4-bcfe-cef963f7f0b3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:36.627978826Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1595,"":"downstairs","":"downstairs","session_id":"ca10a56b-ee54-43e4-bcfe-cef963f7f0b3"} {"msg":"[2]R flush_numbers[0..12]: [178, 181, 181, 175, 177, 179, 175, 175, 181, 181, 180, 178]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:36.627989616Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1595,"":"downstairs","":"downstairs","session_id":"ca10a56b-ee54-43e4-bcfe-cef963f7f0b3"} {"msg":"[2]R generation[0..12]: [110, 113, 113, 110, 110, 111, 110, 110, 113, 113, 112, 110]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:36.628000276Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1595,"":"downstairs","":"downstairs","session_id":"ca10a56b-ee54-43e4-bcfe-cef963f7f0b3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:36.628011546Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1595,"":"downstairs","":"downstairs","session_id":"ca10a56b-ee54-43e4-bcfe-cef963f7f0b3"} {"msg":"Max found gen is 114","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:36.628021076Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1595,"":"downstairs","":"downstairs","session_id":"ca10a56b-ee54-43e4-bcfe-cef963f7f0b3"} {"msg":"Generation requested: 114 >= found:114","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:36.628030876Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1595,"":"downstairs","":"downstairs","session_id":"ca10a56b-ee54-43e4-bcfe-cef963f7f0b3"} {"msg":"Next flush: 182","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:36.628040366Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1595,"":"downstairs","":"downstairs","session_id":"ca10a56b-ee54-43e4-bcfe-cef963f7f0b3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:36.628049796Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1595,"":"downstairs","":"downstairs","session_id":"ca10a56b-ee54-43e4-bcfe-cef963f7f0b3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:36.628059926Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1595,"session_id":"ca10a56b-ee54-43e4-bcfe-cef963f7f0b3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:36.628068866Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1595,"client":"0","":"downstairs","session_id":"ca10a56b-ee54-43e4-bcfe-cef963f7f0b3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:36.628079016Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1595,"client":"1","":"downstairs","session_id":"ca10a56b-ee54-43e4-bcfe-cef963f7f0b3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:36.628089036Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1595,"client":"2","":"downstairs","session_id":"ca10a56b-ee54-43e4-bcfe-cef963f7f0b3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:36.628099346Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1595,"session_id":"ca10a56b-ee54-43e4-bcfe-cef963f7f0b3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:36.628108586Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1595,"session_id":"ca10a56b-ee54-43e4-bcfe-cef963f7f0b3"} {"msg":"12cf73ec-6903-4e28-9f40-3aa52283d92d is now active with session: ca10a56b-ee54-43e4-bcfe-cef963f7f0b3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:36.628118806Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1595,"session_id":"ca10a56b-ee54-43e4-bcfe-cef963f7f0b3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:36.628128886Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1595,"session_id":"ca10a56b-ee54-43e4-bcfe-cef963f7f0b3"} {"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:36.628138076Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1595} {"msg":"Activated sub_volume 12cf73ec-6903-4e28-9f40-3aa52283d92d","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:36.628146246Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1595} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:36.628154506Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1595,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:36.628473316Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1595,"task":"crutest"} 01/30 Write block 366 len 16384 data: 12 9 9 8 02/30 Read block 463 len 36864 03/30 Read block 217 len 40960 04/30 Read block 129 len 8192 05/30 Read block 102 len 8192 06/30 Write block 515 len 36864 data: 7 6 8 9 12 12 12 10 10 07/30 Read block 445 len 4096 08/30 Read block 118 len 12288 09/30 Read block 214 len 24576 10/30 Write block 21 len 16384 data: 12 10 11 10 11/30 Write block 12 len 28672 data: 10 8 9 8 8 9 10 12/30 Write block 8 len 28672 data: 11 9 9 8 11 9 10 13/30 Read block 497 len 28672 14/30 Write block 317 len 20480 data: 8 9 10 12 11 15/30 Write block 133 len 36864 data: 5 6 8 8 8 11 14 13 11 16/30 Read block 435 len 40960 17/30 Read block 63 len 40960 18/30 Read block 415 len 20480 19/30 Read block 395 len 12288 20/30 Write block 557 len 12288 data: 8 6 7 21/30 Read block 513 len 16384 22/30 Write block 594 len 12288 data: 5 6 4 23/30 Write block 78 len 24576 data: 12 11 11 8 9 11 24/30 Read block 123 len 40960 25/30 Read block 428 len 8192 26/30 Write block 241 len 24576 data: 13 11 12 9 11 10 27/30 Read block 202 len 8192 28/30 Write block 339 len 24576 data: 8 10 11 8 8 8 29/30 Read block 477 len 28672 30/30 Read block 471 len 40960 ---------------------------------------------------------------- Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Read 9 Done Done Done false 1002 Acked Read 10 Done Done Done false 1003 Acked Read 2 Done Done Sent false 1004 Acked Read 2 Done Done Sent false 1005 Acked Write 9 Done Done Sent false 1006 Acked Read 1 Done Done Sent false 1007 Acked Read 3 Done Done Sent false 1008 Acked Read 6 Done Done Sent false 1009 Acked Write 4 Done Done Sent false 1010 Acked Write 7 Done Done Sent false 1011 Acked Write 7 Done Done Sent false 1012 Acked Read 7 Done Done Sent false 1013 Acked Write 5 Done Done Sent false 1014 Acked Write 9 Done Done Sent false 1015 Acked Read 10 Done Done Sent false 1016 Acked Read 10 Done Done Sent false 1017 Acked Read 5 Done Done Sent false 1018 Acked Read 3 Done Done Sent false 1019 Acked Write 3 Done Done Sent false 1020 Acked Read 4 Done Done Sent false 1021 Acked Write 3 Done Done Sent false 1022 Acked Write 6 Done Done Sent false 1023 Acked Read 10 Done Done Sent false 1024 Acked Read 2 Done Done Sent false 1025 Acked Write 6 Done Done Sent false 1026 Acked Read 2 Done Done Sent false 1027 Acked Write 6 Done Done Sent false 1028 Acked Read 7 Done Done Sent false 1029 Acked Read 10 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 27 27 Done 30 30 3 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:23:36.637154129Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1595} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:36.63733215Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1595,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 110 110 110 178 178 178 T T F <--- 1 020-039 113 113 113 181 181 181 T T F <--- 2 040-059 113 113 113 181 181 181 F F F 3 060-079 110 110 110 175 175 175 T T F <--- 4 080-099 110 110 110 177 177 177 T T F <--- 5 100-119 111 111 111 179 179 179 F F F 6 120-139 110 110 110 175 175 175 T T F <--- 7 140-159 110 110 110 175 175 175 T T F <--- 8 160-179 113 113 113 181 181 181 F F F 9 180-199 113 113 113 181 181 181 F F F 10 200-219 112 112 112 180 180 180 F F F 11 220-239 110 110 110 178 178 178 F F F 12 240-259 110 110 110 175 175 175 T T F <--- 13 260-279 108 108 108 173 173 173 F F F 14 280-299 106 106 106 170 170 170 F F F 15 300-319 108 108 108 173 173 173 T T F <--- 16 320-339 107 107 107 172 172 172 T T F <--- 17 340-359 106 106 106 168 168 168 T T F <--- 18 360-379 111 111 111 179 179 179 T T T 19 380-399 111 111 111 179 179 179 F F F 20 400-419 113 113 113 181 181 181 F F F 21 420-439 111 111 111 179 179 179 F F F 22 440-459 105 105 105 167 167 167 F F F 23 460-479 112 112 112 180 180 180 F F F 24 480-499 112 112 112 180 180 180 F F F 25 500-519 112 112 112 180 180 180 T T F <--- 26 520-539 112 112 112 180 180 180 T T F <--- 27 540-559 101 101 101 161 161 161 T T F <--- 28 560-579 109 109 109 174 174 174 F F F 29 580-599 106 106 106 168 168 168 T T F <--- Max gen: 113, Max flush: 181 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 57 Verifying data now No change after verify Loop: 57 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 182 182 182 F F F 1 020-039 114 114 114 182 182 182 F F F 2 040-059 113 113 113 181 181 181 F F F 3 060-079 114 114 114 182 182 182 F F F 4 080-099 114 114 114 182 182 182 F F F 5 100-119 111 111 111 179 179 179 F F F 6 120-139 114 114 114 182 182 182 F F F 7 140-159 114 114 114 182 182 182 F F F 8 160-179 113 113 113 181 181 181 F F F 9 180-199 113 113 113 181 181 181 F F F 10 200-219 112 112 112 180 180 180 F F F 11 220-239 110 110 110 178 178 178 F F F 12 240-259 114 114 114 182 182 182 F F F 13 260-279 108 108 108 173 173 173 F F F 14 280-299 106 106 106 170 170 170 F F F 15 300-319 114 114 114 182 182 182 F F F 16 320-339 114 114 114 182 182 182 F F F 17 340-359 114 114 114 182 182 182 F F F 18 360-379 114 114 114 182 182 182 F F F 19 380-399 111 111 111 179 179 179 F F F 20 400-419 113 113 113 181 181 181 F F F 21 420-439 111 111 111 179 179 179 F F F 22 440-459 105 105 105 167 167 167 F F F 23 460-479 112 112 112 180 180 180 F F F 24 480-499 112 112 112 180 180 180 F F F 25 500-519 114 114 114 182 182 182 F F F 26 520-539 114 114 114 182 182 182 F F F 27 540-559 114 114 114 182 182 182 F F F 28 560-579 109 109 109 174 174 174 F F F 29 580-599 114 114 114 182 182 182 F F F Max gen: 114, Max flush: 182 Begin loop 58 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:40.897688448Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1603,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:23:40.904861309Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1603,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:40.904911199Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1603,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:40.911630129Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1603,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:40.91190927Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1603} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:40.91193796Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1603} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:40.9119467Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1603} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:40.91195869Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1603} {"msg":"Crucible 85622194-b439-4131-9ddc-8489a8d34085 has session id: ad7c3649-9824-40c0-b945-a39bc132b08a","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:40.91234119Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1603,"session_id":"ad7c3649-9824-40c0-b945-a39bc132b08a"} {"msg":"Upstairs opts: Upstairs UUID: 85622194-b439-4131-9ddc-8489a8d34085, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:40.9123641Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1603,"session_id":"ad7c3649-9824-40c0-b945-a39bc132b08a"} {"msg":"Crucible stats registered with UUID: 85622194-b439-4131-9ddc-8489a8d34085","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:40.91237623Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1603,"session_id":"ad7c3649-9824-40c0-b945-a39bc132b08a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:23:40.91238526Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1603,"session_id":"ad7c3649-9824-40c0-b945-a39bc132b08a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:40.912527251Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1603,"":"io task","client":"2","":"downstairs","session_id":"ad7c3649-9824-40c0-b945-a39bc132b08a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:40.912544521Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1603,"":"io task","client":"0","":"downstairs","session_id":"ad7c3649-9824-40c0-b945-a39bc132b08a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:40.912554501Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1603,"":"io task","client":"1","":"downstairs","session_id":"ad7c3649-9824-40c0-b945-a39bc132b08a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:40.912702371Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1603,"task":"crutest"} {"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:40.912717961Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1603} {"msg":"85622194-b439-4131-9ddc-8489a8d34085 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:40.912749941Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1603,"session_id":"ad7c3649-9824-40c0-b945-a39bc132b08a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:40.912759461Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1603,"client":"0","":"downstairs","session_id":"ad7c3649-9824-40c0-b945-a39bc132b08a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:40.912769101Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1603,"client":"1","":"downstairs","session_id":"ad7c3649-9824-40c0-b945-a39bc132b08a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:40.912778751Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1603,"client":"2","":"downstairs","session_id":"ad7c3649-9824-40c0-b945-a39bc132b08a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:40.912788271Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1603,"":"io task","client":"2","":"downstairs","session_id":"ad7c3649-9824-40c0-b945-a39bc132b08a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:40.912798301Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1603,"":"io task","client":"0","":"downstairs","session_id":"ad7c3649-9824-40c0-b945-a39bc132b08a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:40.912808311Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1603,"":"io task","client":"1","":"downstairs","session_id":"ad7c3649-9824-40c0-b945-a39bc132b08a"} {"msg":"ds_connection connected from Ok(127.0.0.1:62930)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:40.912866951Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1603,"":"io task","client":"2","":"downstairs","session_id":"ad7c3649-9824-40c0-b945-a39bc132b08a"} {"msg":"ds_connection connected from Ok(127.0.0.1:32779)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:40.912879531Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1603,"":"io task","client":"1","":"downstairs","session_id":"ad7c3649-9824-40c0-b945-a39bc132b08a"} {"msg":"ds_connection connected from Ok(127.0.0.1:54497)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:40.912889971Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1603,"":"io task","client":"0","":"downstairs","session_id":"ad7c3649-9824-40c0-b945-a39bc132b08a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:40.913528272Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1603,"client":"2","":"downstairs","session_id":"ad7c3649-9824-40c0-b945-a39bc132b08a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:40.913554532Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1603,"client":"1","":"downstairs","session_id":"ad7c3649-9824-40c0-b945-a39bc132b08a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:40.913669302Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1603,"client":"0","":"downstairs","session_id":"ad7c3649-9824-40c0-b945-a39bc132b08a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:40.913806963Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1603,"session_id":"ad7c3649-9824-40c0-b945-a39bc132b08a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:40.913823283Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1603,"session_id":"ad7c3649-9824-40c0-b945-a39bc132b08a"} {"msg":"[0]R flush_numbers[0..12]: [182, 182, 181, 182, 182, 179, 182, 182, 181, 181, 180, 178]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:40.913946973Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1603,"":"downstairs","":"downstairs","session_id":"ad7c3649-9824-40c0-b945-a39bc132b08a"} {"msg":"[0]R generation[0..12]: [114, 114, 113, 114, 114, 111, 114, 114, 113, 113, 112, 110]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:40.913958653Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1603,"":"downstairs","":"downstairs","session_id":"ad7c3649-9824-40c0-b945-a39bc132b08a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:40.913969863Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1603,"":"downstairs","":"downstairs","session_id":"ad7c3649-9824-40c0-b945-a39bc132b08a"} {"msg":"[1]R flush_numbers[0..12]: [182, 182, 181, 182, 182, 179, 182, 182, 181, 181, 180, 178]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:40.913980673Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1603,"":"downstairs","":"downstairs","session_id":"ad7c3649-9824-40c0-b945-a39bc132b08a"} {"msg":"[1]R generation[0..12]: [114, 114, 113, 114, 114, 111, 114, 114, 113, 113, 112, 110]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:40.913991363Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1603,"":"downstairs","":"downstairs","session_id":"ad7c3649-9824-40c0-b945-a39bc132b08a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:40.914002353Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1603,"":"downstairs","":"downstairs","session_id":"ad7c3649-9824-40c0-b945-a39bc132b08a"} {"msg":"[2]R flush_numbers[0..12]: [182, 182, 181, 182, 182, 179, 182, 182, 181, 181, 180, 178]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:40.914013133Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1603,"":"downstairs","":"downstairs","session_id":"ad7c3649-9824-40c0-b945-a39bc132b08a"} {"msg":"[2]R generation[0..12]: [114, 114, 113, 114, 114, 111, 114, 114, 113, 113, 112, 110]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:40.914026393Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1603,"":"downstairs","":"downstairs","session_id":"ad7c3649-9824-40c0-b945-a39bc132b08a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:40.914037593Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1603,"":"downstairs","":"downstairs","session_id":"ad7c3649-9824-40c0-b945-a39bc132b08a"} {"msg":"Max found gen is 115","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:40.914047113Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1603,"":"downstairs","":"downstairs","session_id":"ad7c3649-9824-40c0-b945-a39bc132b08a"} {"msg":"Generation requested: 116 >= found:115","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:40.914056983Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1603,"":"downstairs","":"downstairs","session_id":"ad7c3649-9824-40c0-b945-a39bc132b08a"} {"msg":"Next flush: 183","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:40.914066463Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1603,"":"downstairs","":"downstairs","session_id":"ad7c3649-9824-40c0-b945-a39bc132b08a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:40.914075973Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1603,"":"downstairs","":"downstairs","session_id":"ad7c3649-9824-40c0-b945-a39bc132b08a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:40.914085803Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1603,"session_id":"ad7c3649-9824-40c0-b945-a39bc132b08a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:40.914094673Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1603,"client":"0","":"downstairs","session_id":"ad7c3649-9824-40c0-b945-a39bc132b08a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:40.914104433Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1603,"client":"1","":"downstairs","session_id":"ad7c3649-9824-40c0-b945-a39bc132b08a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:40.914114153Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1603,"client":"2","":"downstairs","session_id":"ad7c3649-9824-40c0-b945-a39bc132b08a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:40.914124243Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1603,"session_id":"ad7c3649-9824-40c0-b945-a39bc132b08a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:40.914133433Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1603,"session_id":"ad7c3649-9824-40c0-b945-a39bc132b08a"} {"msg":"85622194-b439-4131-9ddc-8489a8d34085 is now active with session: ad7c3649-9824-40c0-b945-a39bc132b08a","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-07-22T22:23:40.914143733Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1603,"session_id":"ad7c3649-9824-40c0-b945-a39bc132b08a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:40.914157073Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1603,"session_id":"ad7c3649-9824-40c0-b945-a39bc132b08a"} {"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:40.914166333Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1603} {"msg":"Activated sub_volume 85622194-b439-4131-9ddc-8489a8d34085","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:40.914174473Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1603} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:40.914183753Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1603,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:40.914402594Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1603,"task":"crutest"} 01/30 Write block 331 len 28672 data: 11 9 7 7 8 8 8 02/30 Write block 295 len 16384 data: 10 10 9 11 03/30 Read block 161 len 40960 04/30 Read block 479 len 28672 05/30 Write block 185 len 36864 data: 9 8 10 10 9 8 10 12 12 06/30 Read block 264 len 32768 07/30 Write block 363 len 4096 data: 13 08/30 Read block 90 len 32768 09/30 Write block 142 len 16384 data: 8 6 8 7 10/30 Read block 418 len 8192 11/30 Write block 550 len 32768 data: 6 7 8 6 5 7 7 9 12/30 Write block 98 len 16384 data: 8 6 6 5 13/30 Read block 213 len 12288 14/30 Write block 295 len 20480 data: 11 11 10 12 12 15/30 Read block 28 len 20480 16/30 Read block 196 len 36864 17/30 Flush 18/30 Write block 5 len 28672 data: 7 9 9 12 10 10 9 19/30 Write block 188 len 20480 data: 11 10 9 11 13 20/30 Write block 533 len 12288 data: 8 7 8 21/30 Read block 87 len 4096 22/30 Read block 129 len 20480 23/30 Read block 419 len 8192 24/30 Write block 364 len 20480 data: 15 14 13 10 10 25/30 Write block 157 len 12288 data: 10 13 11 26/30 Read block 506 len 36864 27/30 Read block 382 len 40960 28/30 Read block 557 len 40960 29/30 Write block 36 len 32768 data: 5 5 8 9 7 8 8 7 30/30 Read block 339 len 24576 ---------------------------------------------------------------- Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Write 4 Sent Done Done false 1002 Acked Read 10 Sent Done Done false 1003 Acked Read 7 Sent Done Done false 1004 Acked Write 9 Sent Done Done false 1005 Acked Read 8 Sent Done Done false 1006 Acked Write 1 Sent Done Done false 1007 Acked Read 8 Sent Done Done false 1008 Acked Write 4 Sent Done Done false 1009 Acked Read 2 Sent Done Done false 1010 Acked Write 8 Sent Done Done false 1011 Acked Write 4 Sent Done Done false 1012 Acked Read 3 Sent Done Done false 1013 Acked Write 5 Sent Done Done false 1014 Acked Read 5 Sent Done Done false 1015 Acked Read 9 Sent Done Done false 1016 Acked Flush 0 Sent Done Done false 1017 Acked Write 7 Sent Done Done false 1018 Acked Write 5 Sent Done Done false 1019 Acked Write 3 Sent Done Done false 1020 Acked Read 1 Sent Done Done false 1021 Acked Read 5 Sent Done Done false 1022 Acked Read 2 Sent Done Done false 1023 Acked Write 5 Sent Done Done false 1024 Acked Write 3 Sent Done Done false 1025 Acked Read 9 Sent Done Done false 1026 Acked Read 10 Sent Done Done false 1027 Acked Read 10 Sent Done Done false 1028 Acked Write 8 Sent Done Done false 1029 Acked Read 6 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 0 1 30 Done 1 30 29 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:23:40.923290187Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1603} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:40.923477657Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1603,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 182 182 182 F T T <--- 1 020-039 114 114 114 182 182 182 F T T <--- 2 040-059 113 113 113 181 181 181 F T T <--- 3 060-079 114 114 114 182 182 182 F F F 4 080-099 114 116 116 182 183 183 F F F <--- 5 100-119 111 116 116 179 183 183 F F F <--- 6 120-139 114 114 114 182 182 182 F F F 7 140-159 114 116 116 182 183 183 F T T <--- 8 160-179 113 113 113 181 181 181 F F F 9 180-199 113 116 116 181 183 183 F T T <--- 10 200-219 112 112 112 180 180 180 F F F 11 220-239 110 110 110 178 178 178 F F F 12 240-259 114 114 114 182 182 182 F F F 13 260-279 108 108 108 173 173 173 F F F 14 280-299 106 116 116 170 183 183 F F F <--- 15 300-319 114 114 114 182 182 182 F F F 16 320-339 114 116 116 182 183 183 T F F <--- 17 340-359 114 114 114 182 182 182 F F F 18 360-379 114 116 116 182 183 183 F T T <--- 19 380-399 111 111 111 179 179 179 F F F 20 400-419 113 113 113 181 181 181 F F F 21 420-439 111 111 111 179 179 179 F F F 22 440-459 105 105 105 167 167 167 F F F 23 460-479 112 112 112 180 180 180 F F F 24 480-499 112 112 112 180 180 180 F F F 25 500-519 114 114 114 182 182 182 F F F 26 520-539 114 114 114 182 182 182 F T T <--- 27 540-559 114 116 116 182 183 183 F F F <--- 28 560-579 109 109 109 174 174 174 F F F 29 580-599 114 114 114 182 182 182 F F F Max gen: 116, Max flush: 183 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 58 Verifying data now No change after verify Loop: 58 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 117 117 117 184 184 184 F F F 1 020-039 117 117 117 184 184 184 F F F 2 040-059 117 117 117 184 184 184 F F F 3 060-079 114 114 114 182 182 182 F F F 4 080-099 116 116 116 183 183 183 F F F 5 100-119 116 116 116 183 183 183 F F F 6 120-139 114 114 114 182 182 182 F F F 7 140-159 117 117 117 184 184 184 F F F 8 160-179 113 113 113 181 181 181 F F F 9 180-199 117 117 117 184 184 184 F F F 10 200-219 112 112 112 180 180 180 F F F 11 220-239 110 110 110 178 178 178 F F F 12 240-259 114 114 114 182 182 182 F F F 13 260-279 108 108 108 173 173 173 F F F 14 280-299 116 116 116 183 183 183 F F F 15 300-319 114 114 114 182 182 182 F F F 16 320-339 116 116 116 183 183 183 F F F 17 340-359 114 114 114 182 182 182 F F F 18 360-379 117 117 117 184 184 184 F F F 19 380-399 111 111 111 179 179 179 F F F 20 400-419 113 113 113 181 181 181 F F F 21 420-439 111 111 111 179 179 179 F F F 22 440-459 105 105 105 167 167 167 F F F 23 460-479 112 112 112 180 180 180 F F F 24 480-499 112 112 112 180 180 180 F F F 25 500-519 114 114 114 182 182 182 F F F 26 520-539 117 117 117 184 184 184 F F F 27 540-559 116 116 116 183 183 183 F F F 28 560-579 109 109 109 174 174 174 F F F 29 580-599 114 114 114 182 182 182 F F F Max gen: 117, Max flush: 184 Begin loop 59 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:45.180070091Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1611,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:23:45.187389062Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1611,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:45.187442212Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1611,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:45.194132922Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1611,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:45.194405492Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1611} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:45.194431062Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1611} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:45.194439672Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1611} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:45.194451512Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1611} {"msg":"Crucible 329520ce-07c2-4eea-a2fb-21acec0b09fc has session id: 6680eafe-d157-4469-ab4e-f761c647c77f","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:45.194810123Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1611,"session_id":"6680eafe-d157-4469-ab4e-f761c647c77f"} {"msg":"Upstairs opts: Upstairs UUID: 329520ce-07c2-4eea-a2fb-21acec0b09fc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:45.194852423Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1611,"session_id":"6680eafe-d157-4469-ab4e-f761c647c77f"} {"msg":"Crucible stats registered with UUID: 329520ce-07c2-4eea-a2fb-21acec0b09fc","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:45.194863083Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1611,"session_id":"6680eafe-d157-4469-ab4e-f761c647c77f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:23:45.194872363Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1611,"session_id":"6680eafe-d157-4469-ab4e-f761c647c77f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:45.195015113Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1611,"":"io task","client":"1","":"downstairs","session_id":"6680eafe-d157-4469-ab4e-f761c647c77f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:45.195033003Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1611,"":"io task","client":"2","":"downstairs","session_id":"6680eafe-d157-4469-ab4e-f761c647c77f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:45.195043183Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1611,"":"io task","client":"0","":"downstairs","session_id":"6680eafe-d157-4469-ab4e-f761c647c77f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:45.195204673Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1611,"task":"crutest"} {"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:45.195221443Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1611} {"msg":"329520ce-07c2-4eea-a2fb-21acec0b09fc active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:45.195245063Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1611,"session_id":"6680eafe-d157-4469-ab4e-f761c647c77f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:45.195254623Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1611,"client":"0","":"downstairs","session_id":"6680eafe-d157-4469-ab4e-f761c647c77f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:45.195264433Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1611,"client":"1","":"downstairs","session_id":"6680eafe-d157-4469-ab4e-f761c647c77f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:45.195274163Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1611,"client":"2","":"downstairs","session_id":"6680eafe-d157-4469-ab4e-f761c647c77f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:45.195283963Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1611,"":"io task","client":"2","":"downstairs","session_id":"6680eafe-d157-4469-ab4e-f761c647c77f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:45.195293944Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1611,"":"io task","client":"0","":"downstairs","session_id":"6680eafe-d157-4469-ab4e-f761c647c77f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:45.195304004Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1611,"":"io task","client":"1","":"downstairs","session_id":"6680eafe-d157-4469-ab4e-f761c647c77f"} {"msg":"ds_connection connected from Ok(127.0.0.1:46385)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:45.195374354Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1611,"":"io task","client":"1","":"downstairs","session_id":"6680eafe-d157-4469-ab4e-f761c647c77f"} {"msg":"ds_connection connected from Ok(127.0.0.1:41588)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:45.195512984Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1611,"":"io task","client":"0","":"downstairs","session_id":"6680eafe-d157-4469-ab4e-f761c647c77f"} {"msg":"ds_connection connected from Ok(127.0.0.1:64426)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:45.195526544Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1611,"":"io task","client":"2","":"downstairs","session_id":"6680eafe-d157-4469-ab4e-f761c647c77f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:45.195933174Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1611,"client":"1","":"downstairs","session_id":"6680eafe-d157-4469-ab4e-f761c647c77f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:45.196005035Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1611,"client":"2","":"downstairs","session_id":"6680eafe-d157-4469-ab4e-f761c647c77f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:45.196021475Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1611,"client":"0","":"downstairs","session_id":"6680eafe-d157-4469-ab4e-f761c647c77f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:45.196215975Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1611,"session_id":"6680eafe-d157-4469-ab4e-f761c647c77f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:45.196258945Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1611,"session_id":"6680eafe-d157-4469-ab4e-f761c647c77f"} {"msg":"[0]R flush_numbers[0..12]: [184, 184, 184, 182, 183, 183, 182, 184, 181, 184, 180, 178]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:45.196349855Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1611,"":"downstairs","":"downstairs","session_id":"6680eafe-d157-4469-ab4e-f761c647c77f"} {"msg":"[0]R generation[0..12]: [117, 117, 117, 114, 116, 116, 114, 117, 113, 117, 112, 110]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:45.196361295Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1611,"":"downstairs","":"downstairs","session_id":"6680eafe-d157-4469-ab4e-f761c647c77f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:45.196372205Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1611,"":"downstairs","":"downstairs","session_id":"6680eafe-d157-4469-ab4e-f761c647c77f"} {"msg":"[1]R flush_numbers[0..12]: [184, 184, 184, 182, 183, 183, 182, 184, 181, 184, 180, 178]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:45.196383125Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1611,"":"downstairs","":"downstairs","session_id":"6680eafe-d157-4469-ab4e-f761c647c77f"} {"msg":"[1]R generation[0..12]: [117, 117, 117, 114, 116, 116, 114, 117, 113, 117, 112, 110]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:45.196394025Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1611,"":"downstairs","":"downstairs","session_id":"6680eafe-d157-4469-ab4e-f761c647c77f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:45.196404955Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1611,"":"downstairs","":"downstairs","session_id":"6680eafe-d157-4469-ab4e-f761c647c77f"} {"msg":"[2]R flush_numbers[0..12]: [184, 184, 184, 182, 183, 183, 182, 184, 181, 184, 180, 178]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:45.196415705Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1611,"":"downstairs","":"downstairs","session_id":"6680eafe-d157-4469-ab4e-f761c647c77f"} {"msg":"[2]R generation[0..12]: [117, 117, 117, 114, 116, 116, 114, 117, 113, 117, 112, 110]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:45.196426455Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1611,"":"downstairs","":"downstairs","session_id":"6680eafe-d157-4469-ab4e-f761c647c77f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:45.196437455Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1611,"":"downstairs","":"downstairs","session_id":"6680eafe-d157-4469-ab4e-f761c647c77f"} {"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:45.196446915Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1611,"":"downstairs","":"downstairs","session_id":"6680eafe-d157-4469-ab4e-f761c647c77f"} {"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:45.196458475Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1611,"":"downstairs","":"downstairs","session_id":"6680eafe-d157-4469-ab4e-f761c647c77f"} {"msg":"Next flush: 185","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:45.196467865Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1611,"":"downstairs","":"downstairs","session_id":"6680eafe-d157-4469-ab4e-f761c647c77f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:45.196477455Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1611,"":"downstairs","":"downstairs","session_id":"6680eafe-d157-4469-ab4e-f761c647c77f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:45.196487275Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1611,"session_id":"6680eafe-d157-4469-ab4e-f761c647c77f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:45.196496225Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1611,"client":"0","":"downstairs","session_id":"6680eafe-d157-4469-ab4e-f761c647c77f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:45.196505845Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1611,"client":"1","":"downstairs","session_id":"6680eafe-d157-4469-ab4e-f761c647c77f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:45.196515555Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1611,"client":"2","":"downstairs","session_id":"6680eafe-d157-4469-ab4e-f761c647c77f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:45.196525515Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1611,"session_id":"6680eafe-d157-4469-ab4e-f761c647c77f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:45.196534645Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1611,"session_id":"6680eafe-d157-4469-ab4e-f761c647c77f"} {"msg":"329520ce-07c2-4eea-a2fb-21acec0b09fc is now active with session: 6680eafe-d157-4469-ab4e-f761c647c77f","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:45.196544765Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1611,"session_id":"6680eafe-d157-4469-ab4e-f761c647c77f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:45.196553615Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1611,"session_id":"6680eafe-d157-4469-ab4e-f761c647c77f"} {"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:45.196562675Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1611} {"msg":"Activated sub_volume 329520ce-07c2-4eea-a2fb-21acec0b09fc","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:45.196570735Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1611} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:45.196579205Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1611,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:45.196920296Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1611,"task":"crutest"} 01/30 Write block 144 len 28672 data: 9 8 7 8 8 9 9 02/30 Write block 371 len 8192 data: 9 11 03/30 Write block 79 len 36864 data: 12 12 9 10 12 11 13 12 11 04/30 Read block 444 len 12288 05/30 Read block 517 len 4096 06/30 Read block 94 len 12288 07/30 Write block 135 len 40960 data: 9 9 9 12 15 14 12 9 7 10 08/30 Flush 09/30 Write block 4 len 16384 data: 8 8 10 10 10/30 Write block 543 len 8192 data: 8 10 11/30 Write block 413 len 40960 data: 9 9 8 10 10 8 7 7 9 9 12/30 Flush 13/30 Write block 92 len 4096 data: 7 14/30 Flush 15/30 Write block 182 len 24576 data: 5 5 7 10 9 11 16/30 Write block 344 len 12288 data: 9 8 9 17/30 Write block 376 len 16384 data: 14 12 12 14 18/30 Read block 589 len 40960 19/30 Read block 229 len 4096 20/30 Read block 583 len 40960 21/30 Write block 58 len 36864 data: 10 9 9 13 13 12 9 9 9 22/30 Flush 23/30 Write block 550 len 4096 data: 7 24/30 Write block 177 len 12288 data: 6 10 6 25/30 Read block 207 len 8192 26/30 Read block 494 len 32768 27/30 Write block 253 len 36864 data: 8 8 9 7 7 9 9 9 10 28/30 Write block 290 len 20480 data: 8 6 8 11 10 29/30 Read block 415 len 12288 30/30 Write block 363 len 24576 data: 14 16 15 14 11 11 ---------------------------------------------------------------- Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Write 2 Done Done Done false 1002 Acked Write 9 Sent Done Done false 1003 Acked Read 3 Sent Done Done false 1004 Acked Read 1 Sent Done Done false 1005 Acked Read 3 Sent Done Done false 1006 Acked Write 10 Sent Done Done false 1007 Acked Flush 0 Sent Done Done false 1008 Acked Write 4 Sent Done Done false 1009 Acked Write 2 Sent Done Done false 1010 Acked Write 10 Sent Done Done false 1011 Acked Flush 0 Sent Done Done false 1012 Acked Write 1 Sent Done Done false 1013 Acked Flush 0 Sent Done Done false 1014 Acked Write 6 Sent Done Done false 1015 Acked Write 3 Sent Done Done false 1016 Acked Write 4 Sent Done Done false 1017 Acked Read 10 Sent Done Done false 1018 Acked Read 1 Sent Done Done false 1019 Acked Read 10 Sent Done Done false 1020 Acked Write 9 Sent Done Done false 1021 Acked Flush 0 Sent Done Done false 1022 Acked Write 1 Sent Done Done false 1023 Acked Write 3 Sent Done Done false 1024 Acked Read 2 Sent Done Done false 1025 Acked Read 8 Sent Done Done false 1026 Acked Write 9 Sent Done Done false 1027 Acked Write 5 Sent Done Done false 1028 Acked Read 3 Sent Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 1 1 30 Done 2 29 29 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:23:45.205399999Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1611} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:45.205567629Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1611,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 117 118 118 184 186 186 F F F <--- 1 020-039 117 117 117 184 184 184 F F F 2 040-059 117 118 118 184 188 188 F F F <--- 3 060-079 114 118 118 182 188 188 F F F <--- 4 080-099 116 118 118 183 187 187 F F F <--- 5 100-119 116 116 116 183 183 183 F F F 6 120-139 114 118 118 182 185 185 F F F <--- 7 140-159 117 118 118 184 185 185 T F F <--- 8 160-179 113 113 113 181 181 181 F T T <--- 9 180-199 117 118 118 184 188 188 F F F <--- 10 200-219 112 112 112 180 180 180 F F F 11 220-239 110 110 110 178 178 178 F F F 12 240-259 114 114 114 182 182 182 F T T <--- 13 260-279 108 108 108 173 173 173 F T T <--- 14 280-299 116 116 116 183 183 183 F T T <--- 15 300-319 114 114 114 182 182 182 F F F 16 320-339 116 116 116 183 183 183 F F F 17 340-359 114 118 118 182 188 188 F F F <--- 18 360-379 117 118 118 184 188 188 T T T <--- 19 380-399 111 111 111 179 179 179 F F F 20 400-419 113 118 118 181 186 186 F F F <--- 21 420-439 111 118 118 179 186 186 F F F <--- 22 440-459 105 105 105 167 167 167 F F F 23 460-479 112 112 112 180 180 180 F F F 24 480-499 112 112 112 180 180 180 F F F 25 500-519 114 114 114 182 182 182 F F F 26 520-539 117 117 117 184 184 184 F F F 27 540-559 116 118 118 183 186 186 F T T <--- 28 560-579 109 109 109 174 174 174 F F F 29 580-599 114 114 114 182 182 182 F F F Max gen: 118, Max flush: 188 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 59 Verifying data now No change after verify Loop: 59 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 186 186 186 F F F 1 020-039 117 117 117 184 184 184 F F F 2 040-059 118 118 118 188 188 188 F F F 3 060-079 118 118 118 188 188 188 F F F 4 080-099 118 118 118 187 187 187 F F F 5 100-119 116 116 116 183 183 183 F F F 6 120-139 118 118 118 185 185 185 F F F 7 140-159 118 118 118 185 185 185 F F F 8 160-179 119 119 119 189 189 189 F F F 9 180-199 118 118 118 188 188 188 F F F 10 200-219 112 112 112 180 180 180 F F F 11 220-239 110 110 110 178 178 178 F F F 12 240-259 119 119 119 189 189 189 F F F 13 260-279 119 119 119 189 189 189 F F F 14 280-299 119 119 119 189 189 189 F F F 15 300-319 114 114 114 182 182 182 F F F 16 320-339 116 116 116 183 183 183 F F F 17 340-359 118 118 118 188 188 188 F F F 18 360-379 119 119 119 189 189 189 F F F 19 380-399 111 111 111 179 179 179 F F F 20 400-419 118 118 118 186 186 186 F F F 21 420-439 118 118 118 186 186 186 F F F 22 440-459 105 105 105 167 167 167 F F F 23 460-479 112 112 112 180 180 180 F F F 24 480-499 112 112 112 180 180 180 F F F 25 500-519 114 114 114 182 182 182 F F F 26 520-539 117 117 117 184 184 184 F F F 27 540-559 119 119 119 189 189 189 F F F 28 560-579 109 109 109 174 174 174 F F F 29 580-599 114 114 114 182 182 182 F F F Max gen: 119, Max flush: 189 Begin loop 60 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:49.469731674Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1619,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:49.478025276Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1619,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:49.478327377Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1619} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:49.478353207Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1619} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:49.478361887Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1619} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:49.478373947Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1619} {"msg":"Crucible 9b54a09b-a459-4cea-abdc-8698ed774257 has session id: 49bda01c-1fed-4ad3-9dae-8ad346985e4c","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:49.478739337Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1619,"session_id":"49bda01c-1fed-4ad3-9dae-8ad346985e4c"} {"msg":"Upstairs opts: Upstairs UUID: 9b54a09b-a459-4cea-abdc-8698ed774257, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:49.478759777Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1619,"session_id":"49bda01c-1fed-4ad3-9dae-8ad346985e4c"} {"msg":"Crucible stats registered with UUID: 9b54a09b-a459-4cea-abdc-8698ed774257","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:49.478769997Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1619,"session_id":"49bda01c-1fed-4ad3-9dae-8ad346985e4c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:23:49.478779107Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1619,"session_id":"49bda01c-1fed-4ad3-9dae-8ad346985e4c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:49.478920508Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1619,"":"io task","client":"2","":"downstairs","session_id":"49bda01c-1fed-4ad3-9dae-8ad346985e4c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:49.478938148Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1619,"":"io task","client":"0","":"downstairs","session_id":"49bda01c-1fed-4ad3-9dae-8ad346985e4c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:49.478948208Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1619,"":"io task","client":"1","":"downstairs","session_id":"49bda01c-1fed-4ad3-9dae-8ad346985e4c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:49.479123118Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1619,"task":"crutest"} {"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:49.479138788Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1619} {"msg":"9b54a09b-a459-4cea-abdc-8698ed774257 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:49.479155898Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1619,"session_id":"49bda01c-1fed-4ad3-9dae-8ad346985e4c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:49.479166828Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1619,"client":"0","":"downstairs","session_id":"49bda01c-1fed-4ad3-9dae-8ad346985e4c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:49.479176568Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1619,"client":"1","":"downstairs","session_id":"49bda01c-1fed-4ad3-9dae-8ad346985e4c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:49.479186308Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1619,"client":"2","":"downstairs","session_id":"49bda01c-1fed-4ad3-9dae-8ad346985e4c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:49.479195978Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1619,"":"io task","client":"2","":"downstairs","session_id":"49bda01c-1fed-4ad3-9dae-8ad346985e4c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:49.479225068Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1619,"":"io task","client":"0","":"downstairs","session_id":"49bda01c-1fed-4ad3-9dae-8ad346985e4c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:49.479245898Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1619,"":"io task","client":"1","":"downstairs","session_id":"49bda01c-1fed-4ad3-9dae-8ad346985e4c"} {"msg":"ds_connection connected from Ok(127.0.0.1:49045)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:49.479314908Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1619,"":"io task","client":"0","":"downstairs","session_id":"49bda01c-1fed-4ad3-9dae-8ad346985e4c"} {"msg":"ds_connection connected from Ok(127.0.0.1:49495)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:49.479339968Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1619,"":"io task","client":"2","":"downstairs","session_id":"49bda01c-1fed-4ad3-9dae-8ad346985e4c"} {"msg":"ds_connection connected from Ok(127.0.0.1:35017)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:49.479402158Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1619,"":"io task","client":"1","":"downstairs","session_id":"49bda01c-1fed-4ad3-9dae-8ad346985e4c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:49.479982479Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1619,"client":"0","":"downstairs","session_id":"49bda01c-1fed-4ad3-9dae-8ad346985e4c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:49.480035589Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1619,"client":"2","":"downstairs","session_id":"49bda01c-1fed-4ad3-9dae-8ad346985e4c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:49.480048579Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1619,"client":"1","":"downstairs","session_id":"49bda01c-1fed-4ad3-9dae-8ad346985e4c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:49.48031336Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1619,"session_id":"49bda01c-1fed-4ad3-9dae-8ad346985e4c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:49.48036054Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1619,"session_id":"49bda01c-1fed-4ad3-9dae-8ad346985e4c"} {"msg":"[0]R flush_numbers[0..12]: [186, 184, 188, 188, 187, 183, 185, 185, 189, 188, 180, 178]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:49.48040083Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1619,"":"downstairs","":"downstairs","session_id":"49bda01c-1fed-4ad3-9dae-8ad346985e4c"} {"msg":"[0]R generation[0..12]: [118, 117, 118, 118, 118, 116, 118, 118, 119, 118, 112, 110]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:49.48041401Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1619,"":"downstairs","":"downstairs","session_id":"49bda01c-1fed-4ad3-9dae-8ad346985e4c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:49.48042511Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1619,"":"downstairs","":"downstairs","session_id":"49bda01c-1fed-4ad3-9dae-8ad346985e4c"} {"msg":"[1]R flush_numbers[0..12]: [186, 184, 188, 188, 187, 183, 185, 185, 189, 188, 180, 178]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:49.480436Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1619,"":"downstairs","":"downstairs","session_id":"49bda01c-1fed-4ad3-9dae-8ad346985e4c"} {"msg":"[1]R generation[0..12]: [118, 117, 118, 118, 118, 116, 118, 118, 119, 118, 112, 110]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:49.48044692Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1619,"":"downstairs","":"downstairs","session_id":"49bda01c-1fed-4ad3-9dae-8ad346985e4c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:49.4804579Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1619,"":"downstairs","":"downstairs","session_id":"49bda01c-1fed-4ad3-9dae-8ad346985e4c"} {"msg":"[2]R flush_numbers[0..12]: [186, 184, 188, 188, 187, 183, 185, 185, 189, 188, 180, 178]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:49.480469Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1619,"":"downstairs","":"downstairs","session_id":"49bda01c-1fed-4ad3-9dae-8ad346985e4c"} {"msg":"[2]R generation[0..12]: [118, 117, 118, 118, 118, 116, 118, 118, 119, 118, 112, 110]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:49.48047958Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1619,"":"downstairs","":"downstairs","session_id":"49bda01c-1fed-4ad3-9dae-8ad346985e4c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:49.48049063Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1619,"":"downstairs","":"downstairs","session_id":"49bda01c-1fed-4ad3-9dae-8ad346985e4c"} {"msg":"Max found gen is 120","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:49.48050007Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1619,"":"downstairs","":"downstairs","session_id":"49bda01c-1fed-4ad3-9dae-8ad346985e4c"} {"msg":"Generation requested: 120 >= found:120","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:49.48050991Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1619,"":"downstairs","":"downstairs","session_id":"49bda01c-1fed-4ad3-9dae-8ad346985e4c"} {"msg":"Next flush: 190","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:49.4805193Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1619,"":"downstairs","":"downstairs","session_id":"49bda01c-1fed-4ad3-9dae-8ad346985e4c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:49.48052876Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1619,"":"downstairs","":"downstairs","session_id":"49bda01c-1fed-4ad3-9dae-8ad346985e4c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:49.48053859Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1619,"session_id":"49bda01c-1fed-4ad3-9dae-8ad346985e4c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:49.48054743Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1619,"client":"0","":"downstairs","session_id":"49bda01c-1fed-4ad3-9dae-8ad346985e4c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:49.48055747Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1619,"client":"1","":"downstairs","session_id":"49bda01c-1fed-4ad3-9dae-8ad346985e4c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:49.48056843Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1619,"client":"2","":"downstairs","session_id":"49bda01c-1fed-4ad3-9dae-8ad346985e4c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:49.48057845Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1619,"session_id":"49bda01c-1fed-4ad3-9dae-8ad346985e4c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:49.4805877Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1619,"session_id":"49bda01c-1fed-4ad3-9dae-8ad346985e4c"} {"msg":"9b54a09b-a459-4cea-abdc-8698ed774257 is now active with session: 49bda01c-1fed-4ad3-9dae-8ad346985e4c","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:49.48059772Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1619,"session_id":"49bda01c-1fed-4ad3-9dae-8ad346985e4c"} {"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:49.48060693Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1619} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:49.4806145Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1619,"session_id":"49bda01c-1fed-4ad3-9dae-8ad346985e4c"} {"msg":"Activated sub_volume 9b54a09b-a459-4cea-abdc-8698ed774257","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:49.4806237Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1619} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:49.48063179Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1619,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:49.480953941Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1619,"task":"crutest"} 01/30 Write block 112 len 36864 data: 9 10 10 10 10 8 8 7 7 02/30 Write block 462 len 12288 data: 7 10 7 03/30 Read block 452 len 28672 04/30 Read block 567 len 16384 05/30 Write block 23 len 24576 data: 12 11 9 9 11 11 06/30 Write block 14 len 12288 data: 11 9 9 07/30 Flush 08/30 Write block 290 len 36864 data: 9 7 9 12 11 12 12 11 13 09/30 Write block 142 len 8192 data: 10 8 10/30 Read block 493 len 20480 11/30 Flush 12/30 Flush 13/30 Write block 227 len 28672 data: 6 9 6 6 5 6 7 14/30 Read block 114 len 8192 15/30 Write block 183 len 36864 data: 6 8 11 10 12 12 11 10 12 16/30 Read block 339 len 16384 17/30 Read block 422 len 16384 18/30 Write block 557 len 20480 data: 10 7 8 8 9 19/30 Read block 539 len 20480 20/30 Read block 195 len 36864 21/30 Read block 194 len 4096 22/30 Read block 352 len 4096 23/30 Read block 43 len 40960 24/30 Flush 25/30 Write block 171 len 28672 data: 10 10 10 8 6 5 7 26/30 Read block 108 len 16384 27/30 Read block 230 len 20480 28/30 Read block 281 len 32768 29/30 Write block 133 len 28672 data: 6 7 10 10 10 13 16 30/30 Read block 533 len 32768 ---------------------------------------------------------------- Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 7 Done Done Done false 1013 Acked Read 2 Done Done Done false 1014 Acked Write 9 Done Done Done false 1015 Acked Read 4 Done Done Done false 1016 Acked Read 4 Done Done Done false 1017 Acked Write 5 Done Done Done false 1018 Acked Read 5 Done Done Done false 1019 Acked Read 9 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Read 10 Done Sent Done false 1023 Acked Flush 0 Done Sent Done false 1024 Acked Write 7 Done Sent Done false 1025 Acked Read 4 Done Sent Done false 1026 Acked Read 5 Done Sent Done false 1027 Acked Read 8 Done Sent Done false 1028 Acked Write 7 Done Sent Done false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 8 1 9 Done 18 10 17 45 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1011)) Some(JobId(1023)) Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:23:49.490724225Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1619} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:49.490929956Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1619,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 120 120 120 190 190 190 F F F 1 020-039 120 120 120 190 190 190 F F F 2 040-059 118 118 118 188 188 188 F F F 3 060-079 118 118 118 188 188 188 F F F 4 080-099 118 118 118 187 187 187 F F F 5 100-119 120 120 120 190 190 190 F F F 6 120-139 120 120 120 190 190 190 T F T <--- 7 140-159 120 120 120 191 191 191 F F F 8 160-179 119 119 119 189 189 189 T F T <--- 9 180-199 120 118 120 193 188 193 F T F <--- 10 200-219 112 112 112 180 180 180 F F F 11 220-239 120 110 120 193 178 193 F T F <--- 12 240-259 119 119 119 189 189 189 F F F 13 260-279 119 119 119 189 189 189 F F F 14 280-299 120 120 120 191 191 191 F F F 15 300-319 114 114 114 182 182 182 F F F 16 320-339 116 116 116 183 183 183 F F F 17 340-359 118 118 118 188 188 188 F F F 18 360-379 119 119 119 189 189 189 F F F 19 380-399 111 111 111 179 179 179 F F F 20 400-419 118 118 118 186 186 186 F F F 21 420-439 118 118 118 186 186 186 F F F 22 440-459 105 105 105 167 167 167 F F F 23 460-479 120 120 120 190 190 190 F F F 24 480-499 112 112 112 180 180 180 F F F 25 500-519 114 114 114 182 182 182 F F F 26 520-539 117 117 117 184 184 184 F F F 27 540-559 120 119 120 193 189 193 F T F <--- 28 560-579 120 109 120 193 174 193 F T F <--- 29 580-599 114 114 114 182 182 182 F F F Max gen: 120, Max flush: 193 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 60 Verifying data now No change after verify Loop: 60 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 120 120 120 190 190 190 F F F 1 020-039 120 120 120 190 190 190 F F F 2 040-059 118 118 118 188 188 188 F F F 3 060-079 118 118 118 188 188 188 F F F 4 080-099 118 118 118 187 187 187 F F F 5 100-119 120 120 120 190 190 190 F F F 6 120-139 121 121 121 194 194 194 F F F 7 140-159 120 120 120 191 191 191 F F F 8 160-179 121 121 121 194 194 194 F F F 9 180-199 120 120 120 193 193 193 F F F 10 200-219 112 112 112 180 180 180 F F F 11 220-239 120 120 120 193 193 193 F F F 12 240-259 119 119 119 189 189 189 F F F 13 260-279 119 119 119 189 189 189 F F F 14 280-299 120 120 120 191 191 191 F F F 15 300-319 114 114 114 182 182 182 F F F 16 320-339 116 116 116 183 183 183 F F F 17 340-359 118 118 118 188 188 188 F F F 18 360-379 119 119 119 189 189 189 F F F 19 380-399 111 111 111 179 179 179 F F F 20 400-419 118 118 118 186 186 186 F F F 21 420-439 118 118 118 186 186 186 F F F 22 440-459 105 105 105 167 167 167 F F F 23 460-479 120 120 120 190 190 190 F F F 24 480-499 112 112 112 180 180 180 F F F 25 500-519 114 114 114 182 182 182 F F F 26 520-539 117 117 117 184 184 184 F F F 27 540-559 120 120 120 193 193 193 F F F 28 560-579 120 120 120 193 193 193 F F F 29 580-599 114 114 114 182 182 182 F F F Max gen: 121, Max flush: 194 Begin loop 61 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:53.731248085Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1627,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:23:53.738349625Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1627,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:53.738404645Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1627,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:53.745331706Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1627,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:53.745610736Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1627} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:53.745646946Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1627} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:53.745656406Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1627} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:53.745668496Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1627} {"msg":"Crucible 73e377f8-7e68-4400-98cc-c4e50ab31994 has session id: 0f8bf3c0-26f7-4d68-8a64-4b30b4356dc3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:53.746010967Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1627,"session_id":"0f8bf3c0-26f7-4d68-8a64-4b30b4356dc3"} {"msg":"Upstairs opts: Upstairs UUID: 73e377f8-7e68-4400-98cc-c4e50ab31994, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:53.746030697Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1627,"session_id":"0f8bf3c0-26f7-4d68-8a64-4b30b4356dc3"} {"msg":"Crucible stats registered with UUID: 73e377f8-7e68-4400-98cc-c4e50ab31994","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:53.746040977Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1627,"session_id":"0f8bf3c0-26f7-4d68-8a64-4b30b4356dc3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:23:53.746050097Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1627,"session_id":"0f8bf3c0-26f7-4d68-8a64-4b30b4356dc3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:53.746200127Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1627,"":"io task","client":"1","":"downstairs","session_id":"0f8bf3c0-26f7-4d68-8a64-4b30b4356dc3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:53.746217837Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1627,"":"io task","client":"0","":"downstairs","session_id":"0f8bf3c0-26f7-4d68-8a64-4b30b4356dc3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:53.746227897Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1627,"":"io task","client":"2","":"downstairs","session_id":"0f8bf3c0-26f7-4d68-8a64-4b30b4356dc3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:53.746378357Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1627,"task":"crutest"} {"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:53.746390127Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1627} {"msg":"73e377f8-7e68-4400-98cc-c4e50ab31994 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:53.746407497Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1627,"session_id":"0f8bf3c0-26f7-4d68-8a64-4b30b4356dc3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:53.746416477Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1627,"client":"0","":"downstairs","session_id":"0f8bf3c0-26f7-4d68-8a64-4b30b4356dc3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:53.746426237Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1627,"client":"1","":"downstairs","session_id":"0f8bf3c0-26f7-4d68-8a64-4b30b4356dc3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:53.746435987Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1627,"client":"2","":"downstairs","session_id":"0f8bf3c0-26f7-4d68-8a64-4b30b4356dc3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:53.746445567Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1627,"":"io task","client":"2","":"downstairs","session_id":"0f8bf3c0-26f7-4d68-8a64-4b30b4356dc3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:53.746455837Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1627,"":"io task","client":"0","":"downstairs","session_id":"0f8bf3c0-26f7-4d68-8a64-4b30b4356dc3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:53.746465897Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1627,"":"io task","client":"1","":"downstairs","session_id":"0f8bf3c0-26f7-4d68-8a64-4b30b4356dc3"} {"msg":"ds_connection connected from Ok(127.0.0.1:56542)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:53.746534428Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1627,"":"io task","client":"1","":"downstairs","session_id":"0f8bf3c0-26f7-4d68-8a64-4b30b4356dc3"} {"msg":"ds_connection connected from Ok(127.0.0.1:56658)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:53.746561718Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1627,"":"io task","client":"2","":"downstairs","session_id":"0f8bf3c0-26f7-4d68-8a64-4b30b4356dc3"} {"msg":"ds_connection connected from Ok(127.0.0.1:63135)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:53.746573298Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1627,"":"io task","client":"0","":"downstairs","session_id":"0f8bf3c0-26f7-4d68-8a64-4b30b4356dc3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:53.747289119Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1627,"client":"2","":"downstairs","session_id":"0f8bf3c0-26f7-4d68-8a64-4b30b4356dc3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:53.747312929Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1627,"client":"1","":"downstairs","session_id":"0f8bf3c0-26f7-4d68-8a64-4b30b4356dc3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:53.747372699Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1627,"client":"0","":"downstairs","session_id":"0f8bf3c0-26f7-4d68-8a64-4b30b4356dc3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:53.747647729Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1627,"session_id":"0f8bf3c0-26f7-4d68-8a64-4b30b4356dc3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:53.747691399Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1627,"session_id":"0f8bf3c0-26f7-4d68-8a64-4b30b4356dc3"} {"msg":"[0]R flush_numbers[0..12]: [190, 190, 188, 188, 187, 190, 194, 191, 194, 193, 180, 193]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:53.747749399Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1627,"":"downstairs","":"downstairs","session_id":"0f8bf3c0-26f7-4d68-8a64-4b30b4356dc3"} {"msg":"[0]R generation[0..12]: [120, 120, 118, 118, 118, 120, 121, 120, 121, 120, 112, 120]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:53.747760929Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1627,"":"downstairs","":"downstairs","session_id":"0f8bf3c0-26f7-4d68-8a64-4b30b4356dc3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:53.747772029Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1627,"":"downstairs","":"downstairs","session_id":"0f8bf3c0-26f7-4d68-8a64-4b30b4356dc3"} {"msg":"[1]R flush_numbers[0..12]: [190, 190, 188, 188, 187, 190, 194, 191, 194, 193, 180, 193]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:53.747782919Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1627,"":"downstairs","":"downstairs","session_id":"0f8bf3c0-26f7-4d68-8a64-4b30b4356dc3"} {"msg":"[1]R generation[0..12]: [120, 120, 118, 118, 118, 120, 121, 120, 121, 120, 112, 120]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:53.74779369Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1627,"":"downstairs","":"downstairs","session_id":"0f8bf3c0-26f7-4d68-8a64-4b30b4356dc3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:53.7478049Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1627,"":"downstairs","":"downstairs","session_id":"0f8bf3c0-26f7-4d68-8a64-4b30b4356dc3"} {"msg":"[2]R flush_numbers[0..12]: [190, 190, 188, 188, 187, 190, 194, 191, 194, 193, 180, 193]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:53.74781779Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1627,"":"downstairs","":"downstairs","session_id":"0f8bf3c0-26f7-4d68-8a64-4b30b4356dc3"} {"msg":"[2]R generation[0..12]: [120, 120, 118, 118, 118, 120, 121, 120, 121, 120, 112, 120]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:53.74782844Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1627,"":"downstairs","":"downstairs","session_id":"0f8bf3c0-26f7-4d68-8a64-4b30b4356dc3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:53.74783954Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1627,"":"downstairs","":"downstairs","session_id":"0f8bf3c0-26f7-4d68-8a64-4b30b4356dc3"} {"msg":"Max found gen is 122","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:53.7478491Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1627,"":"downstairs","":"downstairs","session_id":"0f8bf3c0-26f7-4d68-8a64-4b30b4356dc3"} {"msg":"Generation requested: 122 >= found:122","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:53.747859Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1627,"":"downstairs","":"downstairs","session_id":"0f8bf3c0-26f7-4d68-8a64-4b30b4356dc3"} {"msg":"Next flush: 195","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:53.74786855Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1627,"":"downstairs","":"downstairs","session_id":"0f8bf3c0-26f7-4d68-8a64-4b30b4356dc3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:53.74787803Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1627,"":"downstairs","":"downstairs","session_id":"0f8bf3c0-26f7-4d68-8a64-4b30b4356dc3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:53.74788796Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1627,"session_id":"0f8bf3c0-26f7-4d68-8a64-4b30b4356dc3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:53.74789687Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1627,"client":"0","":"downstairs","session_id":"0f8bf3c0-26f7-4d68-8a64-4b30b4356dc3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:53.74790666Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1627,"client":"1","":"downstairs","session_id":"0f8bf3c0-26f7-4d68-8a64-4b30b4356dc3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:53.74791638Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1627,"client":"2","":"downstairs","session_id":"0f8bf3c0-26f7-4d68-8a64-4b30b4356dc3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:53.74792621Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1627,"session_id":"0f8bf3c0-26f7-4d68-8a64-4b30b4356dc3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:53.74793543Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1627,"session_id":"0f8bf3c0-26f7-4d68-8a64-4b30b4356dc3"} {"msg":"73e377f8-7e68-4400-98cc-c4e50ab31994 is now active with session: 0f8bf3c0-26f7-4d68-8a64-4b30b4356dc3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:53.74794551Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1627,"session_id":"0f8bf3c0-26f7-4d68-8a64-4b30b4356dc3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:53.74795442Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1627,"session_id":"0f8bf3c0-26f7-4d68-8a64-4b30b4356dc3"} {"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:53.74796389Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1627} {"msg":"Activated sub_volume 73e377f8-7e68-4400-98cc-c4e50ab31994","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:53.7479733Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1627} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:53.7479816Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1627,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:53.7483219Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1627,"task":"crutest"} 01/30 Write block 19 len 40960 data: 11 12 13 11 13 12 10 10 12 12 02/30 Write block 276 len 28672 data: 12 13 10 8 11 9 8 03/30 Write block 552 len 4096 data: 9 04/30 Flush 05/30 Write block 504 len 24576 data: 8 10 10 5 5 6 06/30 Write block 186 len 12288 data: 11 13 13 07/30 Read block 371 len 32768 08/30 Write block 533 len 36864 data: 9 8 9 8 9 9 9 8 8 09/30 Read block 518 len 4096 10/30 Flush 11/30 Write block 456 len 36864 data: 10 9 12 10 9 8 8 11 8 12/30 Write block 132 len 8192 data: 8 7 13/30 Write block 331 len 8192 data: 12 10 14/30 Read block 99 len 32768 15/30 Write block 567 len 20480 data: 12 8 10 11 12 16/30 Read block 21 len 4096 17/30 Write block 283 len 28672 data: 7 5 6 7 7 6 7 18/30 Flush 19/30 Write block 520 len 32768 data: 13 13 11 11 10 14 13 14 20/30 Write block 27 len 28672 data: 13 13 11 13 12 9 7 21/30 Flush 22/30 Flush 23/30 Write block 385 len 12288 data: 9 9 9 24/30 Read block 493 len 8192 25/30 Read block 80 len 24576 26/30 Write block 452 len 20480 data: 6 6 7 9 11 27/30 Write block 302 len 24576 data: 12 11 11 10 10 9 28/30 Write block 461 len 28672 data: 9 9 12 9 7 8 10 29/30 Write block 417 len 28672 data: 11 9 8 8 10 10 8 30/30 Write block 285 len 20480 data: 7 8 8 7 8 ---------------------------------------------------------------- Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Sent Done Done false 1001 Acked Write 7 Sent Done Done false 1002 Acked Write 1 Sent Done Done false 1003 Acked Flush 0 Sent Done Done false 1004 Acked Write 6 Sent Done Done false 1005 Acked Write 3 Sent Done Done false 1006 Acked Read 8 Sent Done Done false 1007 Acked Write 9 Sent Done Done false 1008 Acked Read 1 Sent Done Done false 1009 Acked Flush 0 Sent Done Done false 1010 Acked Write 9 Sent Done Done false 1011 Acked Write 2 Sent Done Done false 1012 Acked Write 2 Sent Done Done false 1013 Acked Read 8 Sent Done Done false 1014 Acked Write 5 Sent Done Done false 1015 Acked Read 1 Sent Done Done false 1016 Acked Write 7 Sent Done Done false 1017 Acked Flush 0 Sent Done Done false 1018 Acked Write 8 Sent Done Done false 1019 Acked Write 7 Sent Done Done false 1020 Acked Flush 0 Sent Done Done false 1021 Acked Flush 0 Sent Done Done false 1022 Acked Write 3 Sent Done Done false 1023 Acked Read 2 Sent Done Done false 1024 Acked Read 6 Sent Done Done false 1025 Acked Write 5 Sent Done Done false 1026 Acked Write 6 Sent Done Done false 1027 Acked Write 7 Sent Done Sent false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 2 3 35 Done 0 28 27 55 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:23:53.756824773Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1627} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:53.756966803Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1627,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 120 122 122 190 195 195 F F F <--- 1 020-039 120 122 122 190 198 198 F F F <--- 2 040-059 118 118 118 188 188 188 F F F 3 060-079 118 118 118 188 188 188 F F F 4 080-099 118 118 118 187 187 187 F F F 5 100-119 120 120 120 190 190 190 F F F 6 120-139 121 122 122 194 197 197 F F F <--- 7 140-159 120 120 120 191 191 191 F F F 8 160-179 121 121 121 194 194 194 F F F 9 180-199 120 122 122 193 196 196 F F F <--- 10 200-219 112 112 112 180 180 180 F F F 11 220-239 120 120 120 193 193 193 F F F 12 240-259 119 119 119 189 189 189 F F F 13 260-279 119 122 122 189 195 195 F F F <--- 14 280-299 120 122 122 191 197 197 F T T <--- 15 300-319 114 114 114 182 182 182 F T T <--- 16 320-339 116 122 122 183 197 197 F F F <--- 17 340-359 118 118 118 188 188 188 F F F 18 360-379 119 119 119 189 189 189 F F F 19 380-399 111 111 111 179 179 179 F T T <--- 20 400-419 118 118 118 186 186 186 F T T <--- 21 420-439 118 118 118 186 186 186 F T T <--- 22 440-459 105 122 122 167 197 197 F T T <--- 23 460-479 120 122 122 190 197 197 F T T <--- 24 480-499 112 112 112 180 180 180 F F F 25 500-519 114 122 122 182 196 196 F F F <--- 26 520-539 117 122 122 184 198 198 F F F <--- 27 540-559 120 122 122 193 196 196 F F F <--- 28 560-579 120 122 122 193 197 197 F F F <--- 29 580-599 114 114 114 182 182 182 F F F Max gen: 122, Max flush: 198 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 61 Verifying data now No change after verify Loop: 61 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 195 195 195 F F F 1 020-039 122 122 122 198 198 198 F F F 2 040-059 118 118 118 188 188 188 F F F 3 060-079 118 118 118 188 188 188 F F F 4 080-099 118 118 118 187 187 187 F F F 5 100-119 120 120 120 190 190 190 F F F 6 120-139 122 122 122 197 197 197 F F F 7 140-159 120 120 120 191 191 191 F F F 8 160-179 121 121 121 194 194 194 F F F 9 180-199 122 122 122 196 196 196 F F F 10 200-219 112 112 112 180 180 180 F F F 11 220-239 120 120 120 193 193 193 F F F 12 240-259 119 119 119 189 189 189 F F F 13 260-279 122 122 122 195 195 195 F F F 14 280-299 123 123 123 199 199 199 F F F 15 300-319 123 123 123 199 199 199 F F F 16 320-339 122 122 122 197 197 197 F F F 17 340-359 118 118 118 188 188 188 F F F 18 360-379 119 119 119 189 189 189 F F F 19 380-399 123 123 123 199 199 199 F F F 20 400-419 123 123 123 199 199 199 F F F 21 420-439 123 123 123 199 199 199 F F F 22 440-459 123 123 123 199 199 199 F F F 23 460-479 123 123 123 199 199 199 F F F 24 480-499 112 112 112 180 180 180 F F F 25 500-519 122 122 122 196 196 196 F F F 26 520-539 122 122 122 198 198 198 F F F 27 540-559 122 122 122 196 196 196 F F F 28 560-579 122 122 122 197 197 197 F F F 29 580-599 114 114 114 182 182 182 F F F Max gen: 123, Max flush: 199 Begin loop 62 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:58.029789451Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1635,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:58.038163623Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1635,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:58.038460254Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1635} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:58.038486924Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1635} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:58.038495584Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1635} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:58.038507584Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1635} {"msg":"Crucible 4d77bddb-41b7-4441-8e4d-0c51b6d78e4c has session id: c56ac497-4455-4260-b2f4-8c178e567d1b","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:58.038874964Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1635,"session_id":"c56ac497-4455-4260-b2f4-8c178e567d1b"} {"msg":"Upstairs opts: Upstairs UUID: 4d77bddb-41b7-4441-8e4d-0c51b6d78e4c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:58.038903924Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1635,"session_id":"c56ac497-4455-4260-b2f4-8c178e567d1b"} {"msg":"Crucible stats registered with UUID: 4d77bddb-41b7-4441-8e4d-0c51b6d78e4c","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:58.038915084Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1635,"session_id":"c56ac497-4455-4260-b2f4-8c178e567d1b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:23:58.038924444Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1635,"session_id":"c56ac497-4455-4260-b2f4-8c178e567d1b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:58.039065215Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1635,"":"io task","client":"2","":"downstairs","session_id":"c56ac497-4455-4260-b2f4-8c178e567d1b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:58.039086745Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1635,"":"io task","client":"0","":"downstairs","session_id":"c56ac497-4455-4260-b2f4-8c178e567d1b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:58.039097015Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1635,"":"io task","client":"1","":"downstairs","session_id":"c56ac497-4455-4260-b2f4-8c178e567d1b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:58.039238725Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1635,"task":"crutest"} {"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:58.039254435Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1635} {"msg":"4d77bddb-41b7-4441-8e4d-0c51b6d78e4c active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:58.039288185Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1635,"session_id":"c56ac497-4455-4260-b2f4-8c178e567d1b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:58.039297635Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1635,"client":"0","":"downstairs","session_id":"c56ac497-4455-4260-b2f4-8c178e567d1b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:58.039307525Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1635,"client":"1","":"downstairs","session_id":"c56ac497-4455-4260-b2f4-8c178e567d1b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:58.039317225Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1635,"client":"2","":"downstairs","session_id":"c56ac497-4455-4260-b2f4-8c178e567d1b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:58.039326955Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1635,"":"io task","client":"2","":"downstairs","session_id":"c56ac497-4455-4260-b2f4-8c178e567d1b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:58.039337105Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1635,"":"io task","client":"0","":"downstairs","session_id":"c56ac497-4455-4260-b2f4-8c178e567d1b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:58.039347045Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1635,"":"io task","client":"1","":"downstairs","session_id":"c56ac497-4455-4260-b2f4-8c178e567d1b"} {"msg":"ds_connection connected from Ok(127.0.0.1:48834)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:58.039385205Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1635,"":"io task","client":"0","":"downstairs","session_id":"c56ac497-4455-4260-b2f4-8c178e567d1b"} {"msg":"ds_connection connected from Ok(127.0.0.1:50781)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:58.039396075Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1635,"":"io task","client":"2","":"downstairs","session_id":"c56ac497-4455-4260-b2f4-8c178e567d1b"} {"msg":"ds_connection connected from Ok(127.0.0.1:57421)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:58.039406435Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1635,"":"io task","client":"1","":"downstairs","session_id":"c56ac497-4455-4260-b2f4-8c178e567d1b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:58.040103026Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1635,"client":"0","":"downstairs","session_id":"c56ac497-4455-4260-b2f4-8c178e567d1b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:58.040129176Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1635,"client":"1","":"downstairs","session_id":"c56ac497-4455-4260-b2f4-8c178e567d1b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:58.040143306Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1635,"client":"2","":"downstairs","session_id":"c56ac497-4455-4260-b2f4-8c178e567d1b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:58.040407497Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1635,"session_id":"c56ac497-4455-4260-b2f4-8c178e567d1b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:58.040442637Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1635,"session_id":"c56ac497-4455-4260-b2f4-8c178e567d1b"} {"msg":"[0]R flush_numbers[0..12]: [195, 198, 188, 188, 187, 190, 197, 191, 194, 196, 180, 193]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:58.040510927Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1635,"":"downstairs","":"downstairs","session_id":"c56ac497-4455-4260-b2f4-8c178e567d1b"} {"msg":"[0]R generation[0..12]: [122, 122, 118, 118, 118, 120, 122, 120, 121, 122, 112, 120]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:58.040522387Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1635,"":"downstairs","":"downstairs","session_id":"c56ac497-4455-4260-b2f4-8c178e567d1b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:58.040533497Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1635,"":"downstairs","":"downstairs","session_id":"c56ac497-4455-4260-b2f4-8c178e567d1b"} {"msg":"[1]R flush_numbers[0..12]: [195, 198, 188, 188, 187, 190, 197, 191, 194, 196, 180, 193]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:58.040544327Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1635,"":"downstairs","":"downstairs","session_id":"c56ac497-4455-4260-b2f4-8c178e567d1b"} {"msg":"[1]R generation[0..12]: [122, 122, 118, 118, 118, 120, 122, 120, 121, 122, 112, 120]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:58.040555067Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1635,"":"downstairs","":"downstairs","session_id":"c56ac497-4455-4260-b2f4-8c178e567d1b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:58.040566127Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1635,"":"downstairs","":"downstairs","session_id":"c56ac497-4455-4260-b2f4-8c178e567d1b"} {"msg":"[2]R flush_numbers[0..12]: [195, 198, 188, 188, 187, 190, 197, 191, 194, 196, 180, 193]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:58.040576817Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1635,"":"downstairs","":"downstairs","session_id":"c56ac497-4455-4260-b2f4-8c178e567d1b"} {"msg":"[2]R generation[0..12]: [122, 122, 118, 118, 118, 120, 122, 120, 121, 122, 112, 120]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:58.040587457Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1635,"":"downstairs","":"downstairs","session_id":"c56ac497-4455-4260-b2f4-8c178e567d1b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:58.040598447Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1635,"":"downstairs","":"downstairs","session_id":"c56ac497-4455-4260-b2f4-8c178e567d1b"} {"msg":"Max found gen is 124","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:58.040608097Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1635,"":"downstairs","":"downstairs","session_id":"c56ac497-4455-4260-b2f4-8c178e567d1b"} {"msg":"Generation requested: 124 >= found:124","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:58.040618007Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1635,"":"downstairs","":"downstairs","session_id":"c56ac497-4455-4260-b2f4-8c178e567d1b"} {"msg":"Next flush: 200","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:58.040628907Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1635,"":"downstairs","":"downstairs","session_id":"c56ac497-4455-4260-b2f4-8c178e567d1b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:58.040638437Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1635,"":"downstairs","":"downstairs","session_id":"c56ac497-4455-4260-b2f4-8c178e567d1b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:58.040648277Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1635,"session_id":"c56ac497-4455-4260-b2f4-8c178e567d1b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:58.040657287Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1635,"client":"0","":"downstairs","session_id":"c56ac497-4455-4260-b2f4-8c178e567d1b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:58.040667137Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1635,"client":"1","":"downstairs","session_id":"c56ac497-4455-4260-b2f4-8c178e567d1b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:58.040676947Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1635,"client":"2","":"downstairs","session_id":"c56ac497-4455-4260-b2f4-8c178e567d1b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:58.040686837Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1635,"session_id":"c56ac497-4455-4260-b2f4-8c178e567d1b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:58.040696187Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1635,"session_id":"c56ac497-4455-4260-b2f4-8c178e567d1b"} {"msg":"4d77bddb-41b7-4441-8e4d-0c51b6d78e4c is now active with session: c56ac497-4455-4260-b2f4-8c178e567d1b","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:58.040706207Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1635,"session_id":"c56ac497-4455-4260-b2f4-8c178e567d1b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:58.040715107Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1635,"session_id":"c56ac497-4455-4260-b2f4-8c178e567d1b"} {"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:58.040724267Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1635} {"msg":"Activated sub_volume 4d77bddb-41b7-4441-8e4d-0c51b6d78e4c","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:58.040732407Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1635} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:58.040740677Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1635,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:58.041035448Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1635,"task":"crutest"} 01/30 Flush 02/30 Write block 380 len 28672 data: 13 13 10 9 9 10 10 03/30 Write block 255 len 32768 data: 10 8 8 10 10 10 11 11 04/30 Read block 372 len 20480 05/30 Write block 240 len 32768 data: 13 14 12 13 10 12 11 11 06/30 Write block 506 len 12288 data: 11 6 6 07/30 Write block 337 len 20480 data: 9 7 9 11 12 08/30 Write block 199 len 16384 data: 7 9 8 8 09/30 Write block 499 len 20480 data: 10 8 8 11 9 10/30 Read block 428 len 24576 11/30 Write block 233 len 8192 data: 8 8 12/30 Read block 177 len 24576 13/30 Read block 424 len 24576 14/30 Write block 415 len 12288 data: 9 11 12 15/30 Read block 227 len 40960 16/30 Write block 321 len 32768 data: 12 12 12 9 10 9 9 10 17/30 Flush 18/30 Write block 86 len 40960 data: 13 12 11 12 11 9 8 7 7 9 19/30 Flush 20/30 Write block 583 len 28672 data: 8 9 8 9 10 10 7 21/30 Write block 487 len 36864 data: 14 11 12 12 9 11 13 11 10 22/30 Read block 399 len 8192 23/30 Read block 284 len 8192 24/30 Read block 22 len 32768 25/30 Read block 352 len 20480 26/30 Write block 438 len 24576 data: 9 9 6 5 6 6 27/30 Write block 366 len 40960 data: 15 12 12 9 8 10 12 9 10 12 28/30 Write block 507 len 40960 data: 7 7 7 7 8 10 11 9 8 7 29/30 Read block 40 len 36864 30/30 Write block 445 len 8192 data: 6 6 ---------------------------------------------------------------- Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 7 Done Done Done false 1002 Acked Write 8 Done Done Done false 1003 Acked Read 5 Done Done Done false 1004 Acked Write 8 Done Done Done false 1005 Acked Write 3 Done Sent Done false 1006 Acked Write 5 Done Sent Done false 1007 Acked Write 4 Done Sent Done false 1008 Acked Write 5 Done Sent Done false 1009 Acked Read 6 Done Sent Done false 1010 Acked Write 2 Done Sent Done false 1011 Acked Read 6 Done Sent Done false 1012 Acked Read 6 Done Sent Done false 1013 Acked Write 3 Done Sent Done false 1014 Acked Read 10 Done Sent Done false 1015 Acked Write 8 Done Sent Done false 1016 Acked Flush 0 Done Sent Done false 1017 Acked Write 10 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Write 7 Done Sent Done false 1020 Acked Write 9 Done Sent Done false 1021 Acked Read 2 Done Sent Done false 1022 Acked Read 2 Done Sent Done false 1023 Acked Read 8 Done Sent Done false 1024 Acked Read 5 Done Sent Done false 1025 Acked Write 6 Done Sent Done false 1026 Acked Write 10 Done Sent Done false 1027 Acked Write 10 Done Sent Done false 1028 Acked Read 9 Done Sent Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 25 1 27 Done 28 4 28 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1000)) Some(JobId(1018)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:23:58.049791481Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1635} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:23:58.049995881Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1635,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 195 195 195 F F F 1 020-039 122 122 122 198 198 198 F F F 2 040-059 118 118 118 188 188 188 F F F 3 060-079 118 118 118 188 188 188 F F F 4 080-099 124 118 124 202 187 202 F F F <--- 5 100-119 120 120 120 190 190 190 F F F 6 120-139 122 122 122 197 197 197 F F F 7 140-159 120 120 120 191 191 191 F F F 8 160-179 121 121 121 194 194 194 F F F 9 180-199 124 122 124 201 196 201 F F F <--- 10 200-219 124 112 124 201 180 201 F F F <--- 11 220-239 124 120 124 201 193 201 F F F <--- 12 240-259 124 119 124 201 189 201 F T F <--- 13 260-279 124 122 124 201 195 201 F T F <--- 14 280-299 123 123 123 199 199 199 F F F 15 300-319 123 123 123 199 199 199 F F F 16 320-339 124 122 124 201 197 201 F F F <--- 17 340-359 124 118 124 201 188 201 F F F <--- 18 360-379 119 119 119 189 189 189 T F T <--- 19 380-399 124 123 124 201 199 201 F T F <--- 20 400-419 124 123 124 201 199 201 F F F <--- 21 420-439 123 123 123 199 199 199 T F T <--- 22 440-459 123 123 123 199 199 199 T F T <--- 23 460-479 123 123 123 199 199 199 F F F 24 480-499 124 112 124 201 180 201 T F T <--- 25 500-519 124 122 124 201 196 201 T F T <--- 26 520-539 122 122 122 198 198 198 F F F 27 540-559 122 122 122 196 196 196 F F F 28 560-579 122 122 122 197 197 197 F F F 29 580-599 114 114 114 182 182 182 T F T <--- Max gen: 124, Max flush: 202 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 62 Verifying data now No change after verify Loop: 62 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 195 195 195 F F F 1 020-039 122 122 122 198 198 198 F F F 2 040-059 118 118 118 188 188 188 F F F 3 060-079 118 118 118 188 188 188 F F F 4 080-099 124 124 124 202 202 202 F F F 5 100-119 120 120 120 190 190 190 F F F 6 120-139 122 122 122 197 197 197 F F F 7 140-159 120 120 120 191 191 191 F F F 8 160-179 121 121 121 194 194 194 F F F 9 180-199 124 124 124 201 201 201 F F F 10 200-219 124 124 124 201 201 201 F F F 11 220-239 124 124 124 201 201 201 F F F 12 240-259 124 124 124 201 201 201 F F F 13 260-279 124 124 124 201 201 201 F F F 14 280-299 123 123 123 199 199 199 F F F 15 300-319 123 123 123 199 199 199 F F F 16 320-339 124 124 124 201 201 201 F F F 17 340-359 124 124 124 201 201 201 F F F 18 360-379 125 125 125 203 203 203 F F F 19 380-399 124 124 124 201 201 201 F F F 20 400-419 124 124 124 201 201 201 F F F 21 420-439 125 125 125 203 203 203 F F F 22 440-459 125 125 125 203 203 203 F F F 23 460-479 123 123 123 199 199 199 F F F 24 480-499 125 125 125 203 203 203 F F F 25 500-519 125 125 125 203 203 203 F F F 26 520-539 122 122 122 198 198 198 F F F 27 540-559 122 122 122 196 196 196 F F F 28 560-579 122 122 122 197 197 197 F F F 29 580-599 125 125 125 203 203 203 F F F Max gen: 125, Max flush: 203 Begin loop 63 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:02.307260006Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1643,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:24:02.314425126Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1643,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:02.314479936Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1643,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:02.321446197Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1643,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:02.321714817Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1643} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:02.321740187Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1643} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:02.321748577Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1643} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:02.321760337Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1643} {"msg":"Crucible cc5a9f46-00a4-4973-a69c-fb50d1cf1301 has session id: 83a1aa4e-4a0b-4f19-8394-5d8182633631","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:02.322129928Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1643,"session_id":"83a1aa4e-4a0b-4f19-8394-5d8182633631"} {"msg":"Upstairs opts: Upstairs UUID: cc5a9f46-00a4-4973-a69c-fb50d1cf1301, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:02.322150738Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1643,"session_id":"83a1aa4e-4a0b-4f19-8394-5d8182633631"} {"msg":"Crucible stats registered with UUID: cc5a9f46-00a4-4973-a69c-fb50d1cf1301","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:02.322161178Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1643,"session_id":"83a1aa4e-4a0b-4f19-8394-5d8182633631"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:24:02.322170648Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1643,"session_id":"83a1aa4e-4a0b-4f19-8394-5d8182633631"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:02.322308858Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1643,"":"io task","client":"0","":"downstairs","session_id":"83a1aa4e-4a0b-4f19-8394-5d8182633631"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:02.322339758Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1643,"":"io task","client":"1","":"downstairs","session_id":"83a1aa4e-4a0b-4f19-8394-5d8182633631"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:02.322350868Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1643,"":"io task","client":"2","":"downstairs","session_id":"83a1aa4e-4a0b-4f19-8394-5d8182633631"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:02.322494279Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1643,"task":"crutest"} {"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:02.322506209Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1643} {"msg":"cc5a9f46-00a4-4973-a69c-fb50d1cf1301 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:02.322523159Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1643,"session_id":"83a1aa4e-4a0b-4f19-8394-5d8182633631"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:02.322532089Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1643,"client":"0","":"downstairs","session_id":"83a1aa4e-4a0b-4f19-8394-5d8182633631"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:02.322541669Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1643,"client":"1","":"downstairs","session_id":"83a1aa4e-4a0b-4f19-8394-5d8182633631"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:02.322551359Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1643,"client":"2","":"downstairs","session_id":"83a1aa4e-4a0b-4f19-8394-5d8182633631"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:02.322561049Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1643,"":"io task","client":"2","":"downstairs","session_id":"83a1aa4e-4a0b-4f19-8394-5d8182633631"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:02.322571169Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1643,"":"io task","client":"0","":"downstairs","session_id":"83a1aa4e-4a0b-4f19-8394-5d8182633631"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:02.322581269Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1643,"":"io task","client":"1","":"downstairs","session_id":"83a1aa4e-4a0b-4f19-8394-5d8182633631"} {"msg":"ds_connection connected from Ok(127.0.0.1:46362)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:02.322696869Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1643,"":"io task","client":"1","":"downstairs","session_id":"83a1aa4e-4a0b-4f19-8394-5d8182633631"} {"msg":"ds_connection connected from Ok(127.0.0.1:35785)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:02.322722139Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1643,"":"io task","client":"2","":"downstairs","session_id":"83a1aa4e-4a0b-4f19-8394-5d8182633631"} {"msg":"ds_connection connected from Ok(127.0.0.1:42816)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:02.322787619Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1643,"":"io task","client":"0","":"downstairs","session_id":"83a1aa4e-4a0b-4f19-8394-5d8182633631"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:02.32337808Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1643,"client":"1","":"downstairs","session_id":"83a1aa4e-4a0b-4f19-8394-5d8182633631"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:02.3234053Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1643,"client":"2","":"downstairs","session_id":"83a1aa4e-4a0b-4f19-8394-5d8182633631"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:02.32348926Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1643,"client":"0","":"downstairs","session_id":"83a1aa4e-4a0b-4f19-8394-5d8182633631"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:02.32368241Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1643,"session_id":"83a1aa4e-4a0b-4f19-8394-5d8182633631"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:02.32369957Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1643,"session_id":"83a1aa4e-4a0b-4f19-8394-5d8182633631"} {"msg":"[0]R flush_numbers[0..12]: [195, 198, 188, 188, 202, 190, 197, 191, 194, 201, 201, 201]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:02.32375528Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1643,"":"downstairs","":"downstairs","session_id":"83a1aa4e-4a0b-4f19-8394-5d8182633631"} {"msg":"[0]R generation[0..12]: [122, 122, 118, 118, 124, 120, 122, 120, 121, 124, 124, 124]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:02.32376664Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1643,"":"downstairs","":"downstairs","session_id":"83a1aa4e-4a0b-4f19-8394-5d8182633631"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:02.32377775Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1643,"":"downstairs","":"downstairs","session_id":"83a1aa4e-4a0b-4f19-8394-5d8182633631"} {"msg":"[1]R flush_numbers[0..12]: [195, 198, 188, 188, 202, 190, 197, 191, 194, 201, 201, 201]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:02.32378856Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1643,"":"downstairs","":"downstairs","session_id":"83a1aa4e-4a0b-4f19-8394-5d8182633631"} {"msg":"[1]R generation[0..12]: [122, 122, 118, 118, 124, 120, 122, 120, 121, 124, 124, 124]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:02.32379939Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1643,"":"downstairs","":"downstairs","session_id":"83a1aa4e-4a0b-4f19-8394-5d8182633631"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:02.323810491Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1643,"":"downstairs","":"downstairs","session_id":"83a1aa4e-4a0b-4f19-8394-5d8182633631"} {"msg":"[2]R flush_numbers[0..12]: [195, 198, 188, 188, 202, 190, 197, 191, 194, 201, 201, 201]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:02.323821681Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1643,"":"downstairs","":"downstairs","session_id":"83a1aa4e-4a0b-4f19-8394-5d8182633631"} {"msg":"[2]R generation[0..12]: [122, 122, 118, 118, 124, 120, 122, 120, 121, 124, 124, 124]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:02.323832491Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1643,"":"downstairs","":"downstairs","session_id":"83a1aa4e-4a0b-4f19-8394-5d8182633631"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:02.323843631Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1643,"":"downstairs","":"downstairs","session_id":"83a1aa4e-4a0b-4f19-8394-5d8182633631"} {"msg":"Max found gen is 126","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:02.323853241Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1643,"":"downstairs","":"downstairs","session_id":"83a1aa4e-4a0b-4f19-8394-5d8182633631"} {"msg":"Generation requested: 126 >= found:126","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:02.323863311Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1643,"":"downstairs","":"downstairs","session_id":"83a1aa4e-4a0b-4f19-8394-5d8182633631"} {"msg":"Next flush: 204","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:02.323872801Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1643,"":"downstairs","":"downstairs","session_id":"83a1aa4e-4a0b-4f19-8394-5d8182633631"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:02.323882481Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1643,"":"downstairs","":"downstairs","session_id":"83a1aa4e-4a0b-4f19-8394-5d8182633631"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:02.323892391Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1643,"session_id":"83a1aa4e-4a0b-4f19-8394-5d8182633631"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:02.323902641Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1643,"client":"0","":"downstairs","session_id":"83a1aa4e-4a0b-4f19-8394-5d8182633631"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:02.323912421Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1643,"client":"1","":"downstairs","session_id":"83a1aa4e-4a0b-4f19-8394-5d8182633631"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:02.323922161Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1643,"client":"2","":"downstairs","session_id":"83a1aa4e-4a0b-4f19-8394-5d8182633631"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:02.323932091Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1643,"session_id":"83a1aa4e-4a0b-4f19-8394-5d8182633631"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:02.323941381Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1643,"session_id":"83a1aa4e-4a0b-4f19-8394-5d8182633631"} {"msg":"cc5a9f46-00a4-4973-a69c-fb50d1cf1301 is now active with session: 83a1aa4e-4a0b-4f19-8394-5d8182633631","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:02.323951611Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1643,"session_id":"83a1aa4e-4a0b-4f19-8394-5d8182633631"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:02.323960561Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1643,"session_id":"83a1aa4e-4a0b-4f19-8394-5d8182633631"} {"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:02.323969811Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1643} {"msg":"Activated sub_volume cc5a9f46-00a4-4973-a69c-fb50d1cf1301","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:02.323978031Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1643} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:02.323986401Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1643,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:02.324271451Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1643,"task":"crutest"} 01/30 Write block 187 len 8192 data: 14 14 02/30 Flush 03/30 Flush 04/30 Write block 529 len 20480 data: 11 8 8 8 10 05/30 Read block 119 len 28672 06/30 Flush 07/30 Write block 440 len 16384 data: 7 6 7 7 08/30 Flush 09/30 Write block 194 len 16384 data: 8 9 9 10 10/30 Write block 206 len 40960 data: 8 8 10 11 11 11 10 8 7 8 11/30 Read block 82 len 40960 12/30 Write block 183 len 4096 data: 7 13/30 Read block 215 len 24576 14/30 Read block 143 len 20480 15/30 Flush 16/30 Write block 460 len 24576 data: 10 10 10 13 10 8 17/30 Flush 18/30 Write block 308 len 16384 data: 6 6 7 6 19/30 Read block 520 len 12288 20/30 Write block 226 len 8192 data: 6 7 21/30 Read block 326 len 32768 22/30 Write block 63 len 8192 data: 13 10 23/30 Flush 24/30 Write block 540 len 32768 data: 9 9 8 9 11 8 8 6 25/30 Read block 104 len 12288 26/30 Flush 27/30 Write block 452 len 40960 data: 7 7 8 10 12 10 13 11 11 11 28/30 Read block 531 len 32768 29/30 Read block 134 len 16384 30/30 Read block 371 len 32768 ---------------------------------------------------------------- Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Flush 0 Sent Done Done false 1002 Acked Flush 0 Sent Done Done false 1003 Acked Write 5 Sent Done Done false 1004 Acked Read 7 Sent Done Done false 1005 Acked Flush 0 Sent Done Done false 1006 Acked Write 4 Sent Done Done false 1007 Acked Flush 0 Sent Done Done false 1008 Acked Write 4 Sent Done Done false 1009 Acked Write 10 Sent Done Done false 1010 Acked Read 10 Sent Done Done false 1011 Acked Write 1 Sent Done Done false 1012 Acked Read 6 Sent Done Done false 1013 Acked Read 5 Sent Done Done false 1014 Acked Flush 0 Sent Done Done false 1015 Acked Write 6 Sent Done Done false 1016 Acked Flush 0 Sent Done Done false 1017 Acked Write 4 Sent Done Done false 1018 Acked Read 3 Sent Done Done false 1019 Acked Write 2 Sent Done Done false 1020 Acked Read 8 Sent Done Done false 1021 Acked Write 2 Sent Done Done false 1022 Acked Flush 0 Sent Done Done false 1023 Acked Write 8 Sent Done Done false 1024 Acked Read 3 Sent Done Done false 1025 Acked Flush 0 Sent Done Done false 1026 Acked Write 10 Sent Done Done false 1027 Acked Read 8 Sent Done Done false 1028 Acked Read 4 Sent Done Done false 1029 Acked Read 8 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 0 0 29 Done 1 30 30 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:24:02.333409055Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1643} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:02.333599305Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1643,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 195 195 195 F F F 1 020-039 122 122 122 198 198 198 F F F 2 040-059 118 118 118 188 188 188 F F F 3 060-079 118 126 126 188 210 210 F F F <--- 4 080-099 124 124 124 202 202 202 F F F 5 100-119 120 120 120 190 190 190 F F F 6 120-139 122 122 122 197 197 197 F F F 7 140-159 120 120 120 191 191 191 F F F 8 160-179 121 121 121 194 194 194 F F F 9 180-199 124 126 126 201 208 208 T F F <--- 10 200-219 124 126 126 201 208 208 F F F <--- 11 220-239 124 126 126 201 210 210 F F F <--- 12 240-259 124 124 124 201 201 201 F F F 13 260-279 124 124 124 201 201 201 F F F 14 280-299 123 123 123 199 199 199 F F F 15 300-319 123 126 126 199 210 210 F F F <--- 16 320-339 124 124 124 201 201 201 F F F 17 340-359 124 124 124 201 201 201 F F F 18 360-379 125 125 125 203 203 203 F F F 19 380-399 124 124 124 201 201 201 F F F 20 400-419 124 124 124 201 201 201 F F F 21 420-439 125 125 125 203 203 203 F F F 22 440-459 125 126 126 203 207 207 F T T <--- 23 460-479 123 126 126 199 209 209 F T T <--- 24 480-499 125 125 125 203 203 203 F F F 25 500-519 125 125 125 203 203 203 F F F 26 520-539 122 126 126 198 206 206 F F F <--- 27 540-559 122 126 126 196 211 211 F F F <--- 28 560-579 122 122 122 197 197 197 F F F 29 580-599 125 125 125 203 203 203 F F F Max gen: 126, Max flush: 211 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 63 Verifying data now No change after verify Loop: 63 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 195 195 195 F F F 1 020-039 122 122 122 198 198 198 F F F 2 040-059 118 118 118 188 188 188 F F F 3 060-079 126 126 126 210 210 210 F F F 4 080-099 124 124 124 202 202 202 F F F 5 100-119 120 120 120 190 190 190 F F F 6 120-139 122 122 122 197 197 197 F F F 7 140-159 120 120 120 191 191 191 F F F 8 160-179 121 121 121 194 194 194 F F F 9 180-199 126 126 126 208 208 208 F F F 10 200-219 126 126 126 208 208 208 F F F 11 220-239 126 126 126 210 210 210 F F F 12 240-259 124 124 124 201 201 201 F F F 13 260-279 124 124 124 201 201 201 F F F 14 280-299 123 123 123 199 199 199 F F F 15 300-319 126 126 126 210 210 210 F F F 16 320-339 124 124 124 201 201 201 F F F 17 340-359 124 124 124 201 201 201 F F F 18 360-379 125 125 125 203 203 203 F F F 19 380-399 124 124 124 201 201 201 F F F 20 400-419 124 124 124 201 201 201 F F F 21 420-439 125 125 125 203 203 203 F F F 22 440-459 127 127 127 212 212 212 F F F 23 460-479 127 127 127 212 212 212 F F F 24 480-499 125 125 125 203 203 203 F F F 25 500-519 125 125 125 203 203 203 F F F 26 520-539 126 126 126 206 206 206 F F F 27 540-559 126 126 126 211 211 211 F F F 28 560-579 122 122 122 197 197 197 F F F 29 580-599 125 125 125 203 203 203 F F F Max gen: 127, Max flush: 212 Begin loop 64 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:06.583778979Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1651,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:06.592093981Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1651,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:06.592379002Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1651} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:06.592407472Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1651} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:06.592416422Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1651} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:06.592428192Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1651} {"msg":"Crucible a41df1f9-e83a-46c4-9ccb-21c4c75869e1 has session id: 5454742f-972f-4ad0-8448-047bdfdf0a16","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:06.592797093Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1651,"session_id":"5454742f-972f-4ad0-8448-047bdfdf0a16"} {"msg":"Upstairs opts: Upstairs UUID: a41df1f9-e83a-46c4-9ccb-21c4c75869e1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:06.592827333Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1651,"session_id":"5454742f-972f-4ad0-8448-047bdfdf0a16"} {"msg":"Crucible stats registered with UUID: a41df1f9-e83a-46c4-9ccb-21c4c75869e1","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:06.592838633Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1651,"session_id":"5454742f-972f-4ad0-8448-047bdfdf0a16"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:24:06.592848063Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1651,"session_id":"5454742f-972f-4ad0-8448-047bdfdf0a16"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:06.592992213Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1651,"":"io task","client":"1","":"downstairs","session_id":"5454742f-972f-4ad0-8448-047bdfdf0a16"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:06.593024633Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1651,"":"io task","client":"0","":"downstairs","session_id":"5454742f-972f-4ad0-8448-047bdfdf0a16"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:06.593035813Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1651,"":"io task","client":"2","":"downstairs","session_id":"5454742f-972f-4ad0-8448-047bdfdf0a16"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:06.593167843Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1651,"task":"crutest"} {"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:06.593183843Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1651} {"msg":"a41df1f9-e83a-46c4-9ccb-21c4c75869e1 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:06.593222133Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1651,"session_id":"5454742f-972f-4ad0-8448-047bdfdf0a16"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:06.593231633Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1651,"client":"0","":"downstairs","session_id":"5454742f-972f-4ad0-8448-047bdfdf0a16"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:06.593241453Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1651,"client":"1","":"downstairs","session_id":"5454742f-972f-4ad0-8448-047bdfdf0a16"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:06.593251113Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1651,"client":"2","":"downstairs","session_id":"5454742f-972f-4ad0-8448-047bdfdf0a16"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:06.593260863Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1651,"":"io task","client":"2","":"downstairs","session_id":"5454742f-972f-4ad0-8448-047bdfdf0a16"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:06.593270933Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1651,"":"io task","client":"0","":"downstairs","session_id":"5454742f-972f-4ad0-8448-047bdfdf0a16"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:06.593282843Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1651,"":"io task","client":"1","":"downstairs","session_id":"5454742f-972f-4ad0-8448-047bdfdf0a16"} {"msg":"ds_connection connected from Ok(127.0.0.1:35827)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:06.593324073Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1651,"":"io task","client":"2","":"downstairs","session_id":"5454742f-972f-4ad0-8448-047bdfdf0a16"} {"msg":"ds_connection connected from Ok(127.0.0.1:61833)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:06.593342723Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1651,"":"io task","client":"0","":"downstairs","session_id":"5454742f-972f-4ad0-8448-047bdfdf0a16"} {"msg":"ds_connection connected from Ok(127.0.0.1:44433)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:06.593353643Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1651,"":"io task","client":"1","":"downstairs","session_id":"5454742f-972f-4ad0-8448-047bdfdf0a16"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:06.593998424Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1651,"client":"2","":"downstairs","session_id":"5454742f-972f-4ad0-8448-047bdfdf0a16"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:06.594023554Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1651,"client":"0","":"downstairs","session_id":"5454742f-972f-4ad0-8448-047bdfdf0a16"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:06.594035344Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1651,"client":"1","":"downstairs","session_id":"5454742f-972f-4ad0-8448-047bdfdf0a16"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:06.594310695Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1651,"session_id":"5454742f-972f-4ad0-8448-047bdfdf0a16"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:06.594336235Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1651,"session_id":"5454742f-972f-4ad0-8448-047bdfdf0a16"} {"msg":"[0]R flush_numbers[0..12]: [195, 198, 188, 210, 202, 190, 197, 191, 194, 208, 208, 210]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:06.594396175Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1651,"":"downstairs","":"downstairs","session_id":"5454742f-972f-4ad0-8448-047bdfdf0a16"} {"msg":"[0]R generation[0..12]: [122, 122, 118, 126, 124, 120, 122, 120, 121, 126, 126, 126]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:06.594407385Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1651,"":"downstairs","":"downstairs","session_id":"5454742f-972f-4ad0-8448-047bdfdf0a16"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:06.594418545Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1651,"":"downstairs","":"downstairs","session_id":"5454742f-972f-4ad0-8448-047bdfdf0a16"} {"msg":"[1]R flush_numbers[0..12]: [195, 198, 188, 210, 202, 190, 197, 191, 194, 208, 208, 210]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:06.594429315Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1651,"":"downstairs","":"downstairs","session_id":"5454742f-972f-4ad0-8448-047bdfdf0a16"} {"msg":"[1]R generation[0..12]: [122, 122, 118, 126, 124, 120, 122, 120, 121, 126, 126, 126]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:06.594439995Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1651,"":"downstairs","":"downstairs","session_id":"5454742f-972f-4ad0-8448-047bdfdf0a16"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:06.594453115Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1651,"":"downstairs","":"downstairs","session_id":"5454742f-972f-4ad0-8448-047bdfdf0a16"} {"msg":"[2]R flush_numbers[0..12]: [195, 198, 188, 210, 202, 190, 197, 191, 194, 208, 208, 210]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:06.594463895Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1651,"":"downstairs","":"downstairs","session_id":"5454742f-972f-4ad0-8448-047bdfdf0a16"} {"msg":"[2]R generation[0..12]: [122, 122, 118, 126, 124, 120, 122, 120, 121, 126, 126, 126]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:06.594474595Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1651,"":"downstairs","":"downstairs","session_id":"5454742f-972f-4ad0-8448-047bdfdf0a16"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:06.594485545Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1651,"":"downstairs","":"downstairs","session_id":"5454742f-972f-4ad0-8448-047bdfdf0a16"} {"msg":"Max found gen is 128","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:06.594495045Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1651,"":"downstairs","":"downstairs","session_id":"5454742f-972f-4ad0-8448-047bdfdf0a16"} {"msg":"Generation requested: 128 >= found:128","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:06.594504795Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1651,"":"downstairs","":"downstairs","session_id":"5454742f-972f-4ad0-8448-047bdfdf0a16"} {"msg":"Next flush: 213","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:06.594514245Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1651,"":"downstairs","":"downstairs","session_id":"5454742f-972f-4ad0-8448-047bdfdf0a16"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:06.594523845Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1651,"":"downstairs","":"downstairs","session_id":"5454742f-972f-4ad0-8448-047bdfdf0a16"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:06.594533705Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1651,"session_id":"5454742f-972f-4ad0-8448-047bdfdf0a16"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:06.594542495Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1651,"client":"0","":"downstairs","session_id":"5454742f-972f-4ad0-8448-047bdfdf0a16"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:06.594552315Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1651,"client":"1","":"downstairs","session_id":"5454742f-972f-4ad0-8448-047bdfdf0a16"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:06.594561965Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1651,"client":"2","":"downstairs","session_id":"5454742f-972f-4ad0-8448-047bdfdf0a16"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:06.594571955Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1651,"session_id":"5454742f-972f-4ad0-8448-047bdfdf0a16"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:06.594581325Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1651,"session_id":"5454742f-972f-4ad0-8448-047bdfdf0a16"} {"msg":"a41df1f9-e83a-46c4-9ccb-21c4c75869e1 is now active with session: 5454742f-972f-4ad0-8448-047bdfdf0a16","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:06.594591475Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1651,"session_id":"5454742f-972f-4ad0-8448-047bdfdf0a16"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:06.594601975Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1651,"session_id":"5454742f-972f-4ad0-8448-047bdfdf0a16"} {"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:06.594611135Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1651} {"msg":"Activated sub_volume a41df1f9-e83a-46c4-9ccb-21c4c75869e1","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:06.594619305Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1651} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:06.594627435Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1651,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:06.594907296Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1651,"task":"crutest"} 01/30 Write block 88 len 4096 data: 12 02/30 Read block 281 len 32768 03/30 Flush 04/30 Flush 05/30 Write block 22 len 32768 data: 12 14 13 11 11 14 14 12 06/30 Write block 62 len 20480 data: 14 14 11 10 10 07/30 Flush 08/30 Write block 383 len 4096 data: 10 09/30 Write block 442 len 28672 data: 8 8 7 7 7 8 8 10/30 Write block 384 len 20480 data: 10 11 11 10 11 11/30 Write block 488 len 16384 data: 12 13 13 10 12/30 Write block 589 len 4096 data: 8 13/30 Flush 14/30 Write block 419 len 12288 data: 9 9 11 15/30 Write block 83 len 40960 data: 13 12 14 14 13 13 13 12 10 9 16/30 Write block 25 len 4096 data: 12 17/30 Write block 15 len 32768 data: 10 10 10 11 12 13 14 13 18/30 Write block 43 len 8192 data: 8 5 19/30 Write block 484 len 28672 data: 8 10 12 15 13 14 14 20/30 Read block 64 len 20480 21/30 Read block 242 len 4096 22/30 Write block 543 len 40960 data: 10 12 9 9 7 8 8 8 8 10 23/30 Write block 249 len 12288 data: 10 9 11 24/30 Read block 44 len 40960 25/30 Read block 566 len 12288 26/30 Read block 535 len 36864 27/30 Read block 351 len 16384 28/30 Write block 339 len 36864 data: 10 12 13 9 9 10 9 10 11 29/30 Read block 506 len 12288 30/30 Write block 464 len 24576 data: 11 9 9 11 12 11 ---------------------------------------------------------------- Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1007 Acked Write 1 Done Done Done false 1008 Acked Write 7 Done Done Done false 1009 Acked Write 5 Done Done Done false 1010 Acked Write 4 Done Done Done false 1011 Acked Write 1 Done Done Done false 1012 Acked Flush 0 Done Sent Done false 1013 Acked Write 3 Done Sent Done false 1014 Acked Write 10 Done Sent Done false 1015 Acked Write 1 Done Sent Done false 1016 Acked Write 8 Done Sent Done false 1017 Acked Write 2 Done Sent Done false 1018 Acked Write 7 Done Sent Done false 1019 Acked Read 5 Done Sent Done false 1020 Acked Read 1 Done Sent Done false 1021 Acked Write 10 Done Sent Done false 1022 Acked Write 3 Done Sent Done false 1023 Acked Read 10 Done Sent Done false 1024 Acked Read 3 Done Sent Done false 1025 Acked Read 9 Done Sent Done false 1026 Acked Read 4 Done Sent Done false 1027 Acked Write 9 Done Sent Done false 1028 Acked Read 3 Done Sent Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 18 1 20 Done 22 5 22 49 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1012)) Some(JobId(1006)) Some(JobId(1012)) Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:24:06.602800348Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1651} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:06.602985828Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1651,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 195 195 195 T F T <--- 1 020-039 128 128 128 215 215 215 T F T <--- 2 040-059 118 118 118 188 188 188 T F T <--- 3 060-079 128 128 128 215 215 215 F F F 4 080-099 128 128 128 213 213 213 T F T <--- 5 100-119 120 120 120 190 190 190 F F F 6 120-139 122 122 122 197 197 197 F F F 7 140-159 120 120 120 191 191 191 F F F 8 160-179 121 121 121 194 194 194 F F F 9 180-199 126 126 126 208 208 208 F F F 10 200-219 126 126 126 208 208 208 F F F 11 220-239 126 126 126 210 210 210 F F F 12 240-259 124 124 124 201 201 201 T F T <--- 13 260-279 124 124 124 201 201 201 F F F 14 280-299 123 123 123 199 199 199 F F F 15 300-319 126 126 126 210 210 210 F F F 16 320-339 124 124 124 201 201 201 T F T <--- 17 340-359 124 124 124 201 201 201 T F T <--- 18 360-379 125 125 125 203 203 203 F F F 19 380-399 128 124 128 216 201 216 F T F <--- 20 400-419 124 124 124 201 201 201 T F T <--- 21 420-439 125 125 125 203 203 203 T F T <--- 22 440-459 128 127 128 216 212 216 F T F <--- 23 460-479 127 127 127 212 212 212 T F T <--- 24 480-499 128 125 128 216 203 216 T T T <--- 25 500-519 125 125 125 203 203 203 F F F 26 520-539 126 126 126 206 206 206 F F F 27 540-559 126 126 126 211 211 211 T F T <--- 28 560-579 122 122 122 197 197 197 F F F 29 580-599 128 125 128 216 203 216 F T F <--- Max gen: 128, Max flush: 216 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 64 Verifying data now No change after verify Loop: 64 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 129 129 129 217 217 217 F F F 1 020-039 129 129 129 217 217 217 F F F 2 040-059 129 129 129 217 217 217 F F F 3 060-079 128 128 128 215 215 215 F F F 4 080-099 129 129 129 217 217 217 F F F 5 100-119 120 120 120 190 190 190 F F F 6 120-139 122 122 122 197 197 197 F F F 7 140-159 120 120 120 191 191 191 F F F 8 160-179 121 121 121 194 194 194 F F F 9 180-199 126 126 126 208 208 208 F F F 10 200-219 126 126 126 208 208 208 F F F 11 220-239 126 126 126 210 210 210 F F F 12 240-259 129 129 129 217 217 217 F F F 13 260-279 124 124 124 201 201 201 F F F 14 280-299 123 123 123 199 199 199 F F F 15 300-319 126 126 126 210 210 210 F F F 16 320-339 129 129 129 217 217 217 F F F 17 340-359 129 129 129 217 217 217 F F F 18 360-379 125 125 125 203 203 203 F F F 19 380-399 128 128 128 216 216 216 F F F 20 400-419 129 129 129 217 217 217 F F F 21 420-439 129 129 129 217 217 217 F F F 22 440-459 128 128 128 216 216 216 F F F 23 460-479 129 129 129 217 217 217 F F F 24 480-499 129 129 129 217 217 217 F F F 25 500-519 125 125 125 203 203 203 F F F 26 520-539 126 126 126 206 206 206 F F F 27 540-559 129 129 129 217 217 217 F F F 28 560-579 122 122 122 197 197 197 F F F 29 580-599 128 128 128 216 216 216 F F F Max gen: 129, Max flush: 217 Begin loop 65 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:10.866593671Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1659,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:10.874910774Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1659,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:10.875243994Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1659} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:10.875269784Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1659} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:10.875278664Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1659} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:10.875290434Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1659} {"msg":"Crucible 3954dcfe-070a-44c0-8b13-632832cfe777 has session id: 4e985c37-3195-4398-a099-ef590a44b560","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:10.875661255Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1659,"session_id":"4e985c37-3195-4398-a099-ef590a44b560"} {"msg":"Upstairs opts: Upstairs UUID: 3954dcfe-070a-44c0-8b13-632832cfe777, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:10.875690195Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1659,"session_id":"4e985c37-3195-4398-a099-ef590a44b560"} {"msg":"Crucible stats registered with UUID: 3954dcfe-070a-44c0-8b13-632832cfe777","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:10.875701435Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1659,"session_id":"4e985c37-3195-4398-a099-ef590a44b560"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:24:10.875710755Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1659,"session_id":"4e985c37-3195-4398-a099-ef590a44b560"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:10.875838155Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1659,"":"io task","client":"0","":"downstairs","session_id":"4e985c37-3195-4398-a099-ef590a44b560"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:10.875858195Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1659,"":"io task","client":"2","":"downstairs","session_id":"4e985c37-3195-4398-a099-ef590a44b560"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:10.875868515Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1659,"":"io task","client":"1","":"downstairs","session_id":"4e985c37-3195-4398-a099-ef590a44b560"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:10.876057806Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1659,"task":"crutest"} {"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:10.876071726Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1659} {"msg":"3954dcfe-070a-44c0-8b13-632832cfe777 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:10.876103146Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1659,"session_id":"4e985c37-3195-4398-a099-ef590a44b560"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:10.876112966Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1659,"client":"0","":"downstairs","session_id":"4e985c37-3195-4398-a099-ef590a44b560"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:10.876122856Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1659,"client":"1","":"downstairs","session_id":"4e985c37-3195-4398-a099-ef590a44b560"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:10.876132826Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1659,"client":"2","":"downstairs","session_id":"4e985c37-3195-4398-a099-ef590a44b560"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:10.876142546Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1659,"":"io task","client":"2","":"downstairs","session_id":"4e985c37-3195-4398-a099-ef590a44b560"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:10.876152596Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1659,"":"io task","client":"0","":"downstairs","session_id":"4e985c37-3195-4398-a099-ef590a44b560"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:10.876171916Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1659,"":"io task","client":"1","":"downstairs","session_id":"4e985c37-3195-4398-a099-ef590a44b560"} {"msg":"ds_connection connected from Ok(127.0.0.1:41913)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:10.876248006Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1659,"":"io task","client":"2","":"downstairs","session_id":"4e985c37-3195-4398-a099-ef590a44b560"} {"msg":"ds_connection connected from Ok(127.0.0.1:52001)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:10.876378596Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1659,"":"io task","client":"1","":"downstairs","session_id":"4e985c37-3195-4398-a099-ef590a44b560"} {"msg":"ds_connection connected from Ok(127.0.0.1:46466)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:10.876403706Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1659,"":"io task","client":"0","":"downstairs","session_id":"4e985c37-3195-4398-a099-ef590a44b560"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:10.876918937Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1659,"client":"0","":"downstairs","session_id":"4e985c37-3195-4398-a099-ef590a44b560"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:10.876974877Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1659,"client":"1","":"downstairs","session_id":"4e985c37-3195-4398-a099-ef590a44b560"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:10.876996667Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1659,"client":"2","":"downstairs","session_id":"4e985c37-3195-4398-a099-ef590a44b560"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:10.877284477Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1659,"session_id":"4e985c37-3195-4398-a099-ef590a44b560"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:10.877304998Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1659,"session_id":"4e985c37-3195-4398-a099-ef590a44b560"} {"msg":"[0]R flush_numbers[0..12]: [217, 217, 217, 215, 217, 190, 197, 191, 194, 208, 208, 210]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:10.877315568Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1659,"":"downstairs","":"downstairs","session_id":"4e985c37-3195-4398-a099-ef590a44b560"} {"msg":"[0]R generation[0..12]: [129, 129, 129, 128, 129, 120, 122, 120, 121, 126, 126, 126]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:10.877326938Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1659,"":"downstairs","":"downstairs","session_id":"4e985c37-3195-4398-a099-ef590a44b560"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:10.877338178Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1659,"":"downstairs","":"downstairs","session_id":"4e985c37-3195-4398-a099-ef590a44b560"} {"msg":"[1]R flush_numbers[0..12]: [217, 217, 217, 215, 217, 190, 197, 191, 194, 208, 208, 210]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:10.877349048Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1659,"":"downstairs","":"downstairs","session_id":"4e985c37-3195-4398-a099-ef590a44b560"} {"msg":"[1]R generation[0..12]: [129, 129, 129, 128, 129, 120, 122, 120, 121, 126, 126, 126]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:10.877359718Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1659,"":"downstairs","":"downstairs","session_id":"4e985c37-3195-4398-a099-ef590a44b560"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:10.877370868Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1659,"":"downstairs","":"downstairs","session_id":"4e985c37-3195-4398-a099-ef590a44b560"} {"msg":"[2]R flush_numbers[0..12]: [217, 217, 217, 215, 217, 190, 197, 191, 194, 208, 208, 210]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:10.877381768Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1659,"":"downstairs","":"downstairs","session_id":"4e985c37-3195-4398-a099-ef590a44b560"} {"msg":"[2]R generation[0..12]: [129, 129, 129, 128, 129, 120, 122, 120, 121, 126, 126, 126]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:10.877392628Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1659,"":"downstairs","":"downstairs","session_id":"4e985c37-3195-4398-a099-ef590a44b560"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:10.877403828Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1659,"":"downstairs","":"downstairs","session_id":"4e985c37-3195-4398-a099-ef590a44b560"} {"msg":"Max found gen is 130","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:10.877413428Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1659,"":"downstairs","":"downstairs","session_id":"4e985c37-3195-4398-a099-ef590a44b560"} {"msg":"Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:10.877425968Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1659,"":"downstairs","":"downstairs","session_id":"4e985c37-3195-4398-a099-ef590a44b560"} {"msg":"Next flush: 218","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:10.877435648Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1659,"":"downstairs","":"downstairs","session_id":"4e985c37-3195-4398-a099-ef590a44b560"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:10.877445378Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1659,"":"downstairs","":"downstairs","session_id":"4e985c37-3195-4398-a099-ef590a44b560"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:10.877455438Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1659,"session_id":"4e985c37-3195-4398-a099-ef590a44b560"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:10.877464408Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1659,"client":"0","":"downstairs","session_id":"4e985c37-3195-4398-a099-ef590a44b560"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:10.877474418Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1659,"client":"1","":"downstairs","session_id":"4e985c37-3195-4398-a099-ef590a44b560"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:10.877484128Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1659,"client":"2","":"downstairs","session_id":"4e985c37-3195-4398-a099-ef590a44b560"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:10.877494198Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1659,"session_id":"4e985c37-3195-4398-a099-ef590a44b560"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:10.877503528Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1659,"session_id":"4e985c37-3195-4398-a099-ef590a44b560"} {"msg":"3954dcfe-070a-44c0-8b13-632832cfe777 is now active with session: 4e985c37-3195-4398-a099-ef590a44b560","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:10.877513928Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1659,"session_id":"4e985c37-3195-4398-a099-ef590a44b560"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:10.877522748Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1659,"session_id":"4e985c37-3195-4398-a099-ef590a44b560"} {"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:10.877532058Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1659} {"msg":"Activated sub_volume 3954dcfe-070a-44c0-8b13-632832cfe777","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:10.877540258Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1659} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:10.877548618Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1659,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:10.877851218Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1659,"task":"crutest"} 01/30 Write block 79 len 8192 data: 13 13 02/30 Read block 151 len 40960 03/30 Flush 04/30 Write block 241 len 16384 data: 15 13 14 11 05/30 Flush 06/30 Write block 33 len 8192 data: 8 6 07/30 Write block 333 len 12288 data: 8 8 9 08/30 Read block 166 len 36864 09/30 Read block 352 len 36864 10/30 Write block 345 len 28672 data: 10 11 12 11 10 7 4 11/30 Read block 245 len 28672 12/30 Flush 13/30 Write block 98 len 4096 data: 9 14/30 Read block 272 len 36864 15/30 Write block 57 len 16384 data: 9 11 10 10 16/30 Read block 366 len 8192 17/30 Read block 518 len 4096 18/30 Flush 19/30 Flush 20/30 Write block 407 len 28672 data: 8 9 8 9 7 9 10 21/30 Write block 347 len 4096 data: 13 22/30 Flush 23/30 Write block 78 len 16384 data: 13 14 14 10 24/30 Read block 170 len 36864 25/30 Read block 421 len 36864 26/30 Read block 560 len 32768 27/30 Write block 88 len 28672 data: 14 14 13 11 10 8 8 28/30 Write block 220 len 16384 data: 7 6 6 6 29/30 Write block 233 len 20480 data: 9 9 8 9 11 30/30 Write block 495 len 4096 data: 11 ---------------------------------------------------------------- Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Sent false 1001 Acked Read 10 Done Done Sent false 1002 Acked Flush 0 Done Done Sent false 1003 Acked Write 4 Done Done Sent false 1004 Acked Flush 0 Done Done Sent false 1005 Acked Write 2 Done Done Sent false 1006 Acked Write 3 Done Done Sent false 1007 Acked Read 9 Done Done Sent false 1008 Acked Read 9 Done Done Sent false 1009 Acked Write 7 Done Done Sent false 1010 Acked Read 7 Done Done Sent false 1011 Acked Flush 0 Done Done Sent false 1012 Acked Write 1 Done Done Sent false 1013 Acked Read 9 Done Done Sent false 1014 Acked Write 4 Done Done Sent false 1015 Acked Read 2 Done Done Sent false 1016 Acked Read 1 Done Done Sent false 1017 Acked Flush 0 Done Done Sent false 1018 Acked Flush 0 Done Done Sent false 1019 Acked Write 7 Done Done Sent false 1020 Acked Write 1 Done Done Sent false 1021 Acked Flush 0 Done Done Sent false 1022 Acked Write 4 Done Done Sent false 1023 Acked Read 9 Done Done Sent false 1024 Acked Read 9 Done Done Sent false 1025 Acked Read 8 Done Done Sent false 1026 Acked Write 7 Sent Sent Sent false 1027 Acked Write 4 Sent Sent Sent false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 4 4 30 38 Done 26 26 0 52 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:24:10.886100721Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1659} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:10.886275491Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1659,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 129 129 129 217 217 217 F F F 1 020-039 130 130 129 220 220 217 F F F <--- 2 040-059 130 130 129 221 221 217 F F F <--- 3 060-079 130 130 128 221 221 215 T T F <--- 4 080-099 130 130 129 221 221 217 T T F <--- 5 100-119 120 120 120 190 190 190 F F F 6 120-139 122 122 122 197 197 197 F F F 7 140-159 120 120 120 191 191 191 F F F 8 160-179 121 121 121 194 194 194 F F F 9 180-199 126 126 126 208 208 208 F F F 10 200-219 126 126 126 208 208 208 F F F 11 220-239 126 126 126 210 210 210 T T F <--- 12 240-259 130 130 129 219 219 217 F F F <--- 13 260-279 124 124 124 201 201 201 F F F 14 280-299 123 123 123 199 199 199 F F F 15 300-319 126 126 126 210 210 210 F F F 16 320-339 130 130 129 220 220 217 F F F <--- 17 340-359 130 130 129 223 223 217 F F F <--- 18 360-379 125 125 125 203 203 203 F F F 19 380-399 128 128 128 216 216 216 F F F 20 400-419 130 130 129 223 223 217 F F F <--- 21 420-439 129 129 129 217 217 217 F F F 22 440-459 128 128 128 216 216 216 F F F 23 460-479 129 129 129 217 217 217 F F F 24 480-499 129 129 129 217 217 217 T T F <--- 25 500-519 125 125 125 203 203 203 F F F 26 520-539 126 126 126 206 206 206 F F F 27 540-559 129 129 129 217 217 217 F F F 28 560-579 122 122 122 197 197 197 F F F 29 580-599 128 128 128 216 216 216 F F F Max gen: 130, Max flush: 223 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 65 Verifying data now No change after verify Loop: 65 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 129 129 129 217 217 217 F F F 1 020-039 130 130 130 220 220 220 F F F 2 040-059 130 130 130 221 221 221 F F F 3 060-079 131 131 131 224 224 224 F F F 4 080-099 131 131 131 224 224 224 F F F 5 100-119 120 120 120 190 190 190 F F F 6 120-139 122 122 122 197 197 197 F F F 7 140-159 120 120 120 191 191 191 F F F 8 160-179 121 121 121 194 194 194 F F F 9 180-199 126 126 126 208 208 208 F F F 10 200-219 126 126 126 208 208 208 F F F 11 220-239 131 131 131 224 224 224 F F F 12 240-259 130 130 130 219 219 219 F F F 13 260-279 124 124 124 201 201 201 F F F 14 280-299 123 123 123 199 199 199 F F F 15 300-319 126 126 126 210 210 210 F F F 16 320-339 130 130 130 220 220 220 F F F 17 340-359 130 130 130 223 223 223 F F F 18 360-379 125 125 125 203 203 203 F F F 19 380-399 128 128 128 216 216 216 F F F 20 400-419 130 130 130 223 223 223 F F F 21 420-439 129 129 129 217 217 217 F F F 22 440-459 128 128 128 216 216 216 F F F 23 460-479 129 129 129 217 217 217 F F F 24 480-499 131 131 131 224 224 224 F F F 25 500-519 125 125 125 203 203 203 F F F 26 520-539 126 126 126 206 206 206 F F F 27 540-559 129 129 129 217 217 217 F F F 28 560-579 122 122 122 197 197 197 F F F 29 580-599 128 128 128 216 216 216 F F F Max gen: 131, Max flush: 224 Begin loop 66 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:15.131134927Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1667,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:15.139228719Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1667,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:15.139518159Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1667} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:15.139550169Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1667} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:15.139559239Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1667} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:15.139571279Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1667} {"msg":"Crucible 4059ef75-5ed8-4561-9b51-1a12452e989c has session id: 7b6f84a7-7a2e-4a57-a84e-634e60648b1c","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:15.13993163Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1667,"session_id":"7b6f84a7-7a2e-4a57-a84e-634e60648b1c"} {"msg":"Upstairs opts: Upstairs UUID: 4059ef75-5ed8-4561-9b51-1a12452e989c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:15.13996216Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1667,"session_id":"7b6f84a7-7a2e-4a57-a84e-634e60648b1c"} {"msg":"Crucible stats registered with UUID: 4059ef75-5ed8-4561-9b51-1a12452e989c","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:15.13997313Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1667,"session_id":"7b6f84a7-7a2e-4a57-a84e-634e60648b1c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:24:15.13998252Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1667,"session_id":"7b6f84a7-7a2e-4a57-a84e-634e60648b1c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:15.14011703Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1667,"":"io task","client":"0","":"downstairs","session_id":"7b6f84a7-7a2e-4a57-a84e-634e60648b1c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:15.14014956Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1667,"":"io task","client":"2","":"downstairs","session_id":"7b6f84a7-7a2e-4a57-a84e-634e60648b1c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:15.14016053Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1667,"":"io task","client":"1","":"downstairs","session_id":"7b6f84a7-7a2e-4a57-a84e-634e60648b1c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:15.14029283Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1667,"task":"crutest"} {"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:15.140308731Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1667} {"msg":"4059ef75-5ed8-4561-9b51-1a12452e989c active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:15.140326031Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1667,"session_id":"7b6f84a7-7a2e-4a57-a84e-634e60648b1c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:15.140335131Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1667,"client":"0","":"downstairs","session_id":"7b6f84a7-7a2e-4a57-a84e-634e60648b1c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:15.140346941Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1667,"client":"1","":"downstairs","session_id":"7b6f84a7-7a2e-4a57-a84e-634e60648b1c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:15.140356811Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1667,"client":"2","":"downstairs","session_id":"7b6f84a7-7a2e-4a57-a84e-634e60648b1c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:15.140366641Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1667,"":"io task","client":"2","":"downstairs","session_id":"7b6f84a7-7a2e-4a57-a84e-634e60648b1c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:15.140395951Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1667,"":"io task","client":"0","":"downstairs","session_id":"7b6f84a7-7a2e-4a57-a84e-634e60648b1c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:15.140406531Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1667,"":"io task","client":"1","":"downstairs","session_id":"7b6f84a7-7a2e-4a57-a84e-634e60648b1c"} {"msg":"ds_connection connected from Ok(127.0.0.1:62335)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:15.140467701Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1667,"":"io task","client":"0","":"downstairs","session_id":"7b6f84a7-7a2e-4a57-a84e-634e60648b1c"} {"msg":"ds_connection connected from Ok(127.0.0.1:47716)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:15.140490861Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1667,"":"io task","client":"2","":"downstairs","session_id":"7b6f84a7-7a2e-4a57-a84e-634e60648b1c"} {"msg":"ds_connection connected from Ok(127.0.0.1:43976)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:15.140502081Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1667,"":"io task","client":"1","":"downstairs","session_id":"7b6f84a7-7a2e-4a57-a84e-634e60648b1c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:15.141160112Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1667,"client":"2","":"downstairs","session_id":"7b6f84a7-7a2e-4a57-a84e-634e60648b1c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:15.141176402Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1667,"client":"0","":"downstairs","session_id":"7b6f84a7-7a2e-4a57-a84e-634e60648b1c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:15.141216442Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1667,"client":"1","":"downstairs","session_id":"7b6f84a7-7a2e-4a57-a84e-634e60648b1c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:15.141493282Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1667,"session_id":"7b6f84a7-7a2e-4a57-a84e-634e60648b1c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:15.141529092Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1667,"session_id":"7b6f84a7-7a2e-4a57-a84e-634e60648b1c"} {"msg":"[0]R flush_numbers[0..12]: [217, 220, 221, 224, 224, 190, 197, 191, 194, 208, 208, 224]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:15.141572812Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1667,"":"downstairs","":"downstairs","session_id":"7b6f84a7-7a2e-4a57-a84e-634e60648b1c"} {"msg":"[0]R generation[0..12]: [129, 130, 130, 131, 131, 120, 122, 120, 121, 126, 126, 131]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:15.141584252Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1667,"":"downstairs","":"downstairs","session_id":"7b6f84a7-7a2e-4a57-a84e-634e60648b1c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:15.141597522Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1667,"":"downstairs","":"downstairs","session_id":"7b6f84a7-7a2e-4a57-a84e-634e60648b1c"} {"msg":"[1]R flush_numbers[0..12]: [217, 220, 221, 224, 224, 190, 197, 191, 194, 208, 208, 224]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:15.141608452Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1667,"":"downstairs","":"downstairs","session_id":"7b6f84a7-7a2e-4a57-a84e-634e60648b1c"} {"msg":"[1]R generation[0..12]: [129, 130, 130, 131, 131, 120, 122, 120, 121, 126, 126, 131]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:15.141619262Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1667,"":"downstairs","":"downstairs","session_id":"7b6f84a7-7a2e-4a57-a84e-634e60648b1c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:15.141630433Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1667,"":"downstairs","":"downstairs","session_id":"7b6f84a7-7a2e-4a57-a84e-634e60648b1c"} {"msg":"[2]R flush_numbers[0..12]: [217, 220, 221, 224, 224, 190, 197, 191, 194, 208, 208, 224]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:15.141641153Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1667,"":"downstairs","":"downstairs","session_id":"7b6f84a7-7a2e-4a57-a84e-634e60648b1c"} {"msg":"[2]R generation[0..12]: [129, 130, 130, 131, 131, 120, 122, 120, 121, 126, 126, 131]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:15.141651883Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1667,"":"downstairs","":"downstairs","session_id":"7b6f84a7-7a2e-4a57-a84e-634e60648b1c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:15.141662953Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1667,"":"downstairs","":"downstairs","session_id":"7b6f84a7-7a2e-4a57-a84e-634e60648b1c"} {"msg":"Max found gen is 132","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:15.141672433Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1667,"":"downstairs","":"downstairs","session_id":"7b6f84a7-7a2e-4a57-a84e-634e60648b1c"} {"msg":"Generation requested: 132 >= found:132","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:15.141682263Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1667,"":"downstairs","":"downstairs","session_id":"7b6f84a7-7a2e-4a57-a84e-634e60648b1c"} {"msg":"Next flush: 225","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:15.141691753Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1667,"":"downstairs","":"downstairs","session_id":"7b6f84a7-7a2e-4a57-a84e-634e60648b1c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:15.141701363Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1667,"":"downstairs","":"downstairs","session_id":"7b6f84a7-7a2e-4a57-a84e-634e60648b1c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:15.141711523Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1667,"session_id":"7b6f84a7-7a2e-4a57-a84e-634e60648b1c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:15.141720583Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1667,"client":"0","":"downstairs","session_id":"7b6f84a7-7a2e-4a57-a84e-634e60648b1c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:15.141730593Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1667,"client":"1","":"downstairs","session_id":"7b6f84a7-7a2e-4a57-a84e-634e60648b1c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:15.141740623Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1667,"client":"2","":"downstairs","session_id":"7b6f84a7-7a2e-4a57-a84e-634e60648b1c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:15.141751953Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1667,"session_id":"7b6f84a7-7a2e-4a57-a84e-634e60648b1c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:15.141761133Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1667,"session_id":"7b6f84a7-7a2e-4a57-a84e-634e60648b1c"} {"msg":"4059ef75-5ed8-4561-9b51-1a12452e989c is now active with session: 7b6f84a7-7a2e-4a57-a84e-634e60648b1c","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:15.141771323Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1667,"session_id":"7b6f84a7-7a2e-4a57-a84e-634e60648b1c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:15.141780133Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1667,"session_id":"7b6f84a7-7a2e-4a57-a84e-634e60648b1c"} {"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:15.141789313Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1667} {"msg":"Activated sub_volume 4059ef75-5ed8-4561-9b51-1a12452e989c","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:15.141797243Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1667} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:15.141805533Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1667,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:15.142092313Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1667,"task":"crutest"} 01/30 Write block 331 len 8192 data: 13 11 02/30 Read block 585 len 4096 03/30 Read block 111 len 32768 04/30 Write block 177 len 40960 data: 8 11 7 7 7 6 8 9 12 12 05/30 Read block 11 len 8192 06/30 Read block 224 len 12288 07/30 Flush 08/30 Write block 271 len 28672 data: 14 15 9 11 12 13 14 09/30 Read block 511 len 16384 10/30 Read block 386 len 32768 11/30 Read block 372 len 8192 12/30 Write block 568 len 32768 data: 9 11 12 13 12 12 12 13 13/30 Write block 337 len 36864 data: 10 8 11 13 14 10 10 11 11 14/30 Read block 494 len 12288 15/30 Write block 482 len 16384 data: 6 7 9 11 16/30 Write block 168 len 4096 data: 11 17/30 Read block 34 len 32768 18/30 Write block 383 len 16384 data: 11 11 12 12 19/30 Flush 20/30 Write block 91 len 32768 data: 12 11 9 9 10 8 9 10 21/30 Read block 529 len 16384 22/30 Read block 167 len 32768 23/30 Read block 47 len 4096 24/30 Read block 297 len 24576 25/30 Read block 186 len 20480 26/30 Read block 39 len 20480 27/30 Write block 31 len 8192 data: 13 10 28/30 Read block 137 len 8192 29/30 Write block 66 len 12288 data: 11 10 8 30/30 Read block 273 len 24576 ---------------------------------------------------------------- Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Read 1 Done Sent Done false 1002 Acked Read 8 Done Sent Done false 1003 Acked Write 10 Done Sent Done false 1004 Acked Read 2 Done Sent Done false 1005 Acked Read 3 Done Sent Done false 1006 Acked Flush 0 Done Sent Done false 1007 Acked Write 7 Done Sent Done false 1008 Acked Read 4 Done Sent Done false 1009 Acked Read 8 Done Sent Done false 1010 Acked Read 2 Done Sent Done false 1011 Acked Write 8 Done Sent Done false 1012 Acked Write 9 Done Sent Done false 1013 Acked Read 3 Done Sent Done false 1014 Acked Write 4 Done Sent Done false 1015 Acked Write 1 Done Sent Done false 1016 Acked Read 8 Done Sent Done false 1017 Acked Write 4 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Write 8 Done Sent Done false 1020 Acked Read 4 Done Sent Done false 1021 Acked Read 8 Done Sent Done false 1022 Acked Read 1 Done Sent Done false 1023 Acked Read 6 Done Sent Done false 1024 Acked Read 5 Done Sent Done false 1025 Acked Read 5 Done Sent Done false 1026 Acked Write 2 Done Sent Done false 1027 Acked Read 2 Done Sent Done false 1028 Acked Write 3 Done Sent Done false 1029 Acked Read 6 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 29 0 29 Done 30 1 30 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) None Some(JobId(1018)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:24:15.150759546Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1667} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:15.150944187Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1667,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 129 129 129 217 217 217 F F F 1 020-039 130 130 130 220 220 220 T F T <--- 2 040-059 130 130 130 221 221 221 F F F 3 060-079 131 131 131 224 224 224 T F T <--- 4 080-099 131 131 131 224 224 224 T F T <--- 5 100-119 120 120 120 190 190 190 F F F 6 120-139 122 122 122 197 197 197 F F F 7 140-159 120 120 120 191 191 191 F F F 8 160-179 132 121 132 226 194 226 F F F <--- 9 180-199 132 126 132 225 208 225 F F F <--- 10 200-219 126 126 126 208 208 208 F F F 11 220-239 131 131 131 224 224 224 F F F 12 240-259 130 130 130 219 219 219 F F F 13 260-279 132 124 132 226 201 226 F F F <--- 14 280-299 123 123 123 199 199 199 F F F 15 300-319 126 126 126 210 210 210 F F F 16 320-339 132 130 132 226 220 226 F T F <--- 17 340-359 132 130 132 226 223 226 F F F <--- 18 360-379 125 125 125 203 203 203 F F F 19 380-399 132 128 132 226 216 226 F F F <--- 20 400-419 130 130 130 223 223 223 F F F 21 420-439 129 129 129 217 217 217 F F F 22 440-459 128 128 128 216 216 216 F F F 23 460-479 129 129 129 217 217 217 F F F 24 480-499 132 131 132 226 224 226 F F F <--- 25 500-519 125 125 125 203 203 203 F F F 26 520-539 126 126 126 206 206 206 F F F 27 540-559 129 129 129 217 217 217 F F F 28 560-579 132 122 132 226 197 226 F F F <--- 29 580-599 128 128 128 216 216 216 F F F Max gen: 132, Max flush: 226 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 66 Verifying data now No change after verify Loop: 66 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 129 129 129 217 217 217 F F F 1 020-039 133 133 133 227 227 227 F F F 2 040-059 130 130 130 221 221 221 F F F 3 060-079 133 133 133 227 227 227 F F F 4 080-099 133 133 133 227 227 227 F F F 5 100-119 120 120 120 190 190 190 F F F 6 120-139 122 122 122 197 197 197 F F F 7 140-159 120 120 120 191 191 191 F F F 8 160-179 132 132 132 226 226 226 F F F 9 180-199 132 132 132 225 225 225 F F F 10 200-219 126 126 126 208 208 208 F F F 11 220-239 131 131 131 224 224 224 F F F 12 240-259 130 130 130 219 219 219 F F F 13 260-279 132 132 132 226 226 226 F F F 14 280-299 123 123 123 199 199 199 F F F 15 300-319 126 126 126 210 210 210 F F F 16 320-339 132 132 132 226 226 226 F F F 17 340-359 132 132 132 226 226 226 F F F 18 360-379 125 125 125 203 203 203 F F F 19 380-399 132 132 132 226 226 226 F F F 20 400-419 130 130 130 223 223 223 F F F 21 420-439 129 129 129 217 217 217 F F F 22 440-459 128 128 128 216 216 216 F F F 23 460-479 129 129 129 217 217 217 F F F 24 480-499 132 132 132 226 226 226 F F F 25 500-519 125 125 125 203 203 203 F F F 26 520-539 126 126 126 206 206 206 F F F 27 540-559 129 129 129 217 217 217 F F F 28 560-579 132 132 132 226 226 226 F F F 29 580-599 128 128 128 216 216 216 F F F Max gen: 133, Max flush: 227 Begin loop 67 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:19.396357693Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1675,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:19.404681186Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1675,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:19.405000876Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1675} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:19.405050826Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1675} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:19.405060016Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1675} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:19.405072056Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1675} {"msg":"Crucible dd6e55d5-0001-4060-a2b6-be7be3b7dc98 has session id: 5be5572a-5335-49df-a81b-a2c989a4cb88","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:19.405401437Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1675,"session_id":"5be5572a-5335-49df-a81b-a2c989a4cb88"} {"msg":"Upstairs opts: Upstairs UUID: dd6e55d5-0001-4060-a2b6-be7be3b7dc98, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:19.405423367Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1675,"session_id":"5be5572a-5335-49df-a81b-a2c989a4cb88"} {"msg":"Crucible stats registered with UUID: dd6e55d5-0001-4060-a2b6-be7be3b7dc98","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:19.405433567Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1675,"session_id":"5be5572a-5335-49df-a81b-a2c989a4cb88"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:24:19.405442707Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1675,"session_id":"5be5572a-5335-49df-a81b-a2c989a4cb88"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:19.405596257Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1675,"":"io task","client":"1","":"downstairs","session_id":"5be5572a-5335-49df-a81b-a2c989a4cb88"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:19.405626507Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1675,"":"io task","client":"0","":"downstairs","session_id":"5be5572a-5335-49df-a81b-a2c989a4cb88"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:19.405637467Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1675,"":"io task","client":"2","":"downstairs","session_id":"5be5572a-5335-49df-a81b-a2c989a4cb88"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:19.405761117Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1675,"task":"crutest"} {"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:19.405776527Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1675} {"msg":"dd6e55d5-0001-4060-a2b6-be7be3b7dc98 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:19.405816217Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1675,"session_id":"5be5572a-5335-49df-a81b-a2c989a4cb88"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:19.405825807Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1675,"client":"0","":"downstairs","session_id":"5be5572a-5335-49df-a81b-a2c989a4cb88"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:19.405835477Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1675,"client":"1","":"downstairs","session_id":"5be5572a-5335-49df-a81b-a2c989a4cb88"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:19.405849807Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1675,"client":"2","":"downstairs","session_id":"5be5572a-5335-49df-a81b-a2c989a4cb88"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:19.405859387Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1675,"":"io task","client":"2","":"downstairs","session_id":"5be5572a-5335-49df-a81b-a2c989a4cb88"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:19.405869417Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1675,"":"io task","client":"0","":"downstairs","session_id":"5be5572a-5335-49df-a81b-a2c989a4cb88"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:19.405879647Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1675,"":"io task","client":"1","":"downstairs","session_id":"5be5572a-5335-49df-a81b-a2c989a4cb88"} {"msg":"ds_connection connected from Ok(127.0.0.1:43956)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:19.405946807Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1675,"":"io task","client":"2","":"downstairs","session_id":"5be5572a-5335-49df-a81b-a2c989a4cb88"} {"msg":"ds_connection connected from Ok(127.0.0.1:46593)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:19.405973878Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1675,"":"io task","client":"1","":"downstairs","session_id":"5be5572a-5335-49df-a81b-a2c989a4cb88"} {"msg":"ds_connection connected from Ok(127.0.0.1:39367)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:19.405989978Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1675,"":"io task","client":"0","":"downstairs","session_id":"5be5572a-5335-49df-a81b-a2c989a4cb88"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:19.406639069Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1675,"client":"0","":"downstairs","session_id":"5be5572a-5335-49df-a81b-a2c989a4cb88"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:19.406664149Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1675,"client":"1","":"downstairs","session_id":"5be5572a-5335-49df-a81b-a2c989a4cb88"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:19.406768689Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1675,"client":"2","":"downstairs","session_id":"5be5572a-5335-49df-a81b-a2c989a4cb88"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:19.406924599Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1675,"session_id":"5be5572a-5335-49df-a81b-a2c989a4cb88"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:19.406941839Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1675,"session_id":"5be5572a-5335-49df-a81b-a2c989a4cb88"} {"msg":"[0]R flush_numbers[0..12]: [217, 227, 221, 227, 227, 190, 197, 191, 226, 225, 208, 224]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:19.407041549Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1675,"":"downstairs","":"downstairs","session_id":"5be5572a-5335-49df-a81b-a2c989a4cb88"} {"msg":"[0]R generation[0..12]: [129, 133, 130, 133, 133, 120, 122, 120, 132, 132, 126, 131]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:19.407053059Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1675,"":"downstairs","":"downstairs","session_id":"5be5572a-5335-49df-a81b-a2c989a4cb88"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:19.407064139Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1675,"":"downstairs","":"downstairs","session_id":"5be5572a-5335-49df-a81b-a2c989a4cb88"} {"msg":"[1]R flush_numbers[0..12]: [217, 227, 221, 227, 227, 190, 197, 191, 226, 225, 208, 224]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:19.407074949Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1675,"":"downstairs","":"downstairs","session_id":"5be5572a-5335-49df-a81b-a2c989a4cb88"} {"msg":"[1]R generation[0..12]: [129, 133, 130, 133, 133, 120, 122, 120, 132, 132, 126, 131]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:19.407085729Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1675,"":"downstairs","":"downstairs","session_id":"5be5572a-5335-49df-a81b-a2c989a4cb88"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:19.407096859Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1675,"":"downstairs","":"downstairs","session_id":"5be5572a-5335-49df-a81b-a2c989a4cb88"} {"msg":"[2]R flush_numbers[0..12]: [217, 227, 221, 227, 227, 190, 197, 191, 226, 225, 208, 224]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:19.407107469Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1675,"":"downstairs","":"downstairs","session_id":"5be5572a-5335-49df-a81b-a2c989a4cb88"} {"msg":"[2]R generation[0..12]: [129, 133, 130, 133, 133, 120, 122, 120, 132, 132, 126, 131]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:19.407120779Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1675,"":"downstairs","":"downstairs","session_id":"5be5572a-5335-49df-a81b-a2c989a4cb88"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:19.407131909Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1675,"":"downstairs","":"downstairs","session_id":"5be5572a-5335-49df-a81b-a2c989a4cb88"} {"msg":"Max found gen is 134","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:19.407141519Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1675,"":"downstairs","":"downstairs","session_id":"5be5572a-5335-49df-a81b-a2c989a4cb88"} {"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:19.407151389Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1675,"":"downstairs","":"downstairs","session_id":"5be5572a-5335-49df-a81b-a2c989a4cb88"} {"msg":"Next flush: 228","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:19.407160789Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1675,"":"downstairs","":"downstairs","session_id":"5be5572a-5335-49df-a81b-a2c989a4cb88"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:19.407170339Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1675,"":"downstairs","":"downstairs","session_id":"5be5572a-5335-49df-a81b-a2c989a4cb88"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:19.407180119Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1675,"session_id":"5be5572a-5335-49df-a81b-a2c989a4cb88"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:19.407188999Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1675,"client":"0","":"downstairs","session_id":"5be5572a-5335-49df-a81b-a2c989a4cb88"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:19.407198839Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1675,"client":"1","":"downstairs","session_id":"5be5572a-5335-49df-a81b-a2c989a4cb88"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:19.407208499Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1675,"client":"2","":"downstairs","session_id":"5be5572a-5335-49df-a81b-a2c989a4cb88"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:19.407218509Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1675,"session_id":"5be5572a-5335-49df-a81b-a2c989a4cb88"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:19.407227859Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1675,"session_id":"5be5572a-5335-49df-a81b-a2c989a4cb88"} {"msg":"dd6e55d5-0001-4060-a2b6-be7be3b7dc98 is now active with session: 5be5572a-5335-49df-a81b-a2c989a4cb88","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:19.407237809Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1675,"session_id":"5be5572a-5335-49df-a81b-a2c989a4cb88"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:19.407246709Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1675,"session_id":"5be5572a-5335-49df-a81b-a2c989a4cb88"} {"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:19.407255929Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1675} {"msg":"Activated sub_volume dd6e55d5-0001-4060-a2b6-be7be3b7dc98","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:19.407263979Z","hostname":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 w-01KY5YGK9V792E6WY6KV93YHRV","pid":1675} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:19.40727894Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1675,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:19.40753621Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1675,"task":"crutest"} 01/30 Write block 302 len 16384 data: 13 12 12 11 02/30 Read block 335 len 16384 03/30 Write block 3 len 4096 data: 8 04/30 Write block 494 len 40960 data: 12 12 10 10 9 11 9 9 12 10 05/30 Write block 390 len 12288 data: 9 9 10 06/30 Write block 294 len 36864 data: 12 13 13 12 14 13 12 14 14 07/30 Write block 499 len 4096 data: 12 08/30 Read block 549 len 28672 09/30 Read block 334 len 4096 10/30 Write block 295 len 40960 data: 14 14 13 15 14 13 15 15 13 13 11/30 Read block 439 len 4096 12/30 Read block 234 len 4096 13/30 Flush 14/30 Write block 89 len 24576 data: 15 14 13 12 10 10 15/30 Write block 524 len 36864 data: 11 15 14 15 15 12 9 9 9 16/30 Write block 237 len 16384 data: 12 11 14 14 17/30 Write block 455 len 8192 data: 11 13 18/30 Write block 64 len 16384 data: 12 11 12 11 19/30 Read block 15 len 12288 20/30 Flush 21/30 Flush 22/30 Write block 168 len 4096 data: 12 23/30 Write block 168 len 4096 data: 13 24/30 Write block 111 len 24576 data: 10 10 11 11 11 11 25/30 Read block 334 len 40960 26/30 Read block 207 len 32768 27/30 Write block 580 len 20480 data: 8 8 8 9 10 28/30 Write block 170 len 40960 data: 10 11 11 11 9 7 6 9 12 8 29/30 Read block 115 len 28672 30/30 Read block 272 len 8192 ---------------------------------------------------------------- Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Read 4 Done Done Done false 1002 Acked Write 1 Done Done Done false 1003 Acked Write 10 Done Done Done false 1004 Acked Write 3 Done Done Done false 1005 Acked Write 9 Done Sent Done false 1006 Acked Write 1 Done Sent Done false 1007 Acked Read 7 Done Sent Done false 1008 Acked Read 1 Done Sent Done false 1009 Acked Write 10 Done Sent Done false 1010 Acked Read 1 Done Sent Done false 1011 Acked Read 1 Done Sent Done false 1012 Acked Flush 0 Done Sent Done false 1013 Acked Write 6 Done Sent Done false 1014 Acked Write 9 Done Sent Done false 1015 Acked Write 4 Done Sent Done false 1016 Acked Write 2 Done Sent Done false 1017 Acked Write 4 Done Sent Done false 1018 Acked Read 3 Done Sent Done false 1019 Acked Flush 0 Done Sent Done false 1020 Acked Flush 0 Done Sent Done false 1021 Acked Write 1 Done Sent Done false 1022 Acked Write 1 Done Sent Done false 1023 Acked Write 6 Done Sent Done false 1024 Acked Read 10 Done Sent Done false 1025 Acked Read 8 Done Sent Done false 1026 Acked Write 5 Done Sent Done false 1027 Acked Write 10 Done Sent Done false 1028 Acked Read 7 Done Sent Done false 1029 Acked Read 2 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 25 0 25 Done 30 5 30 65 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) None Some(JobId(1020)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:24:19.415468152Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1675} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:19.415659492Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1675,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 134 129 134 228 217 228 F T F <--- 1 020-039 133 133 133 227 227 227 F F F 2 040-059 130 130 130 221 221 221 F F F 3 060-079 134 133 134 229 227 229 F F F <--- 4 080-099 134 133 134 229 227 229 F F F <--- 5 100-119 120 120 120 190 190 190 T F T <--- 6 120-139 122 122 122 197 197 197 F F F 7 140-159 120 120 120 191 191 191 F F F 8 160-179 132 132 132 226 226 226 T F T <--- 9 180-199 132 132 132 225 225 225 F F F 10 200-219 126 126 126 208 208 208 F F F 11 220-239 134 131 134 229 224 229 F F F <--- 12 240-259 134 130 134 229 219 229 F F F <--- 13 260-279 132 132 132 226 226 226 F F F 14 280-299 134 123 134 228 199 228 F F F <--- 15 300-319 134 126 134 228 210 228 F T F <--- 16 320-339 132 132 132 226 226 226 F F F 17 340-359 132 132 132 226 226 226 F F F 18 360-379 125 125 125 203 203 203 F F F 19 380-399 134 132 134 228 226 228 F T F <--- 20 400-419 130 130 130 223 223 223 F F F 21 420-439 129 129 129 217 217 217 F F F 22 440-459 134 128 134 229 216 229 F F F <--- 23 460-479 129 129 129 217 217 217 F F F 24 480-499 134 132 134 228 226 228 F T F <--- 25 500-519 134 125 134 228 203 228 F T F <--- 26 520-539 134 126 134 229 206 229 F F F <--- 27 540-559 129 129 129 217 217 217 F F F 28 560-579 132 132 132 226 226 226 F F F 29 580-599 128 128 128 216 216 216 T F T <--- Max gen: 134, Max flush: 229 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 67 Verifying data now No change after verify Loop: 67 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 134 134 134 228 228 228 F F F 1 020-039 133 133 133 227 227 227 F F F 2 040-059 130 130 130 221 221 221 F F F 3 060-079 134 134 134 229 229 229 F F F 4 080-099 134 134 134 229 229 229 F F F 5 100-119 135 135 135 230 230 230 F F F 6 120-139 122 122 122 197 197 197 F F F 7 140-159 120 120 120 191 191 191 F F F 8 160-179 135 135 135 230 230 230 F F F 9 180-199 132 132 132 225 225 225 F F F 10 200-219 126 126 126 208 208 208 F F F 11 220-239 134 134 134 229 229 229 F F F 12 240-259 134 134 134 229 229 229 F F F 13 260-279 132 132 132 226 226 226 F F F 14 280-299 134 134 134 228 228 228 F F F 15 300-319 134 134 134 228 228 228 F F F 16 320-339 132 132 132 226 226 226 F F F 17 340-359 132 132 132 226 226 226 F F F 18 360-379 125 125 125 203 203 203 F F F 19 380-399 134 134 134 228 228 228 F F F 20 400-419 130 130 130 223 223 223 F F F 21 420-439 129 129 129 217 217 217 F F F 22 440-459 134 134 134 229 229 229 F F F 23 460-479 129 129 129 217 217 217 F F F 24 480-499 134 134 134 228 228 228 F F F 25 500-519 134 134 134 228 228 228 F F F 26 520-539 134 134 134 229 229 229 F F F 27 540-559 129 129 129 217 217 217 F F F 28 560-579 132 132 132 226 226 226 F F F 29 580-599 135 135 135 230 230 230 F F F Max gen: 135, Max flush: 230 Begin loop 68 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:23.670021522Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1683,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:24:23.677143393Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1683,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:23.677199773Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1683,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:23.684079103Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1683,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:23.684351274Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1683} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:23.684379634Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1683} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:23.684388054Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1683} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:23.684399754Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1683} {"msg":"Crucible 68191ccf-4ed6-4af7-833b-d754062aa735 has session id: b3a7dd17-b723-471c-83ec-ee1f3eec6ab1","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:23.684809524Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1683,"session_id":"b3a7dd17-b723-471c-83ec-ee1f3eec6ab1"} {"msg":"Upstairs opts: Upstairs UUID: 68191ccf-4ed6-4af7-833b-d754062aa735, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:23.684833954Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1683,"session_id":"b3a7dd17-b723-471c-83ec-ee1f3eec6ab1"} {"msg":"Crucible stats registered with UUID: 68191ccf-4ed6-4af7-833b-d754062aa735","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:23.684843864Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1683,"session_id":"b3a7dd17-b723-471c-83ec-ee1f3eec6ab1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:24:23.684852974Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1683,"session_id":"b3a7dd17-b723-471c-83ec-ee1f3eec6ab1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:23.684992865Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1683,"":"io task","client":"0","":"downstairs","session_id":"b3a7dd17-b723-471c-83ec-ee1f3eec6ab1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:23.685033035Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1683,"":"io task","client":"1","":"downstairs","session_id":"b3a7dd17-b723-471c-83ec-ee1f3eec6ab1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:23.685044135Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1683,"":"io task","client":"2","":"downstairs","session_id":"b3a7dd17-b723-471c-83ec-ee1f3eec6ab1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:23.685165595Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1683,"task":"crutest"} {"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:23.685184795Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1683} {"msg":"68191ccf-4ed6-4af7-833b-d754062aa735 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:23.685209445Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1683,"session_id":"b3a7dd17-b723-471c-83ec-ee1f3eec6ab1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:23.685219055Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1683,"client":"0","":"downstairs","session_id":"b3a7dd17-b723-471c-83ec-ee1f3eec6ab1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:23.685228785Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1683,"client":"1","":"downstairs","session_id":"b3a7dd17-b723-471c-83ec-ee1f3eec6ab1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:23.685238545Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1683,"client":"2","":"downstairs","session_id":"b3a7dd17-b723-471c-83ec-ee1f3eec6ab1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:23.685248245Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1683,"":"io task","client":"2","":"downstairs","session_id":"b3a7dd17-b723-471c-83ec-ee1f3eec6ab1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:23.685258325Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1683,"":"io task","client":"0","":"downstairs","session_id":"b3a7dd17-b723-471c-83ec-ee1f3eec6ab1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:23.685268835Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1683,"":"io task","client":"1","":"downstairs","session_id":"b3a7dd17-b723-471c-83ec-ee1f3eec6ab1"} {"msg":"ds_connection connected from Ok(127.0.0.1:37858)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:23.685341865Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1683,"":"io task","client":"0","":"downstairs","session_id":"b3a7dd17-b723-471c-83ec-ee1f3eec6ab1"} {"msg":"ds_connection connected from Ok(127.0.0.1:50646)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:23.685368415Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1683,"":"io task","client":"2","":"downstairs","session_id":"b3a7dd17-b723-471c-83ec-ee1f3eec6ab1"} {"msg":"ds_connection connected from Ok(127.0.0.1:53163)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:23.685480605Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1683,"":"io task","client":"1","":"downstairs","session_id":"b3a7dd17-b723-471c-83ec-ee1f3eec6ab1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:23.685986186Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1683,"client":"1","":"downstairs","session_id":"b3a7dd17-b723-471c-83ec-ee1f3eec6ab1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:23.686010636Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1683,"client":"2","":"downstairs","session_id":"b3a7dd17-b723-471c-83ec-ee1f3eec6ab1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:23.686125346Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1683,"client":"0","":"downstairs","session_id":"b3a7dd17-b723-471c-83ec-ee1f3eec6ab1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:23.686303807Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1683,"session_id":"b3a7dd17-b723-471c-83ec-ee1f3eec6ab1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:23.686315647Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1683,"session_id":"b3a7dd17-b723-471c-83ec-ee1f3eec6ab1"} {"msg":"[0]R flush_numbers[0..12]: [228, 227, 221, 229, 229, 230, 197, 191, 230, 225, 208, 229]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:23.686469257Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1683,"":"downstairs","":"downstairs","session_id":"b3a7dd17-b723-471c-83ec-ee1f3eec6ab1"} {"msg":"[0]R generation[0..12]: [134, 133, 130, 134, 134, 135, 122, 120, 135, 132, 126, 134]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:23.686480847Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1683,"":"downstairs","":"downstairs","session_id":"b3a7dd17-b723-471c-83ec-ee1f3eec6ab1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:23.686492127Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1683,"":"downstairs","":"downstairs","session_id":"b3a7dd17-b723-471c-83ec-ee1f3eec6ab1"} {"msg":"[1]R flush_numbers[0..12]: [228, 227, 221, 229, 229, 230, 197, 191, 230, 225, 208, 229]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:23.686503107Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1683,"":"downstairs","":"downstairs","session_id":"b3a7dd17-b723-471c-83ec-ee1f3eec6ab1"} {"msg":"[1]R generation[0..12]: [134, 133, 130, 134, 134, 135, 122, 120, 135, 132, 126, 134]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:23.686513897Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1683,"":"downstairs","":"downstairs","session_id":"b3a7dd17-b723-471c-83ec-ee1f3eec6ab1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:23.686525057Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1683,"":"downstairs","":"downstairs","session_id":"b3a7dd17-b723-471c-83ec-ee1f3eec6ab1"} {"msg":"[2]R flush_numbers[0..12]: [228, 227, 221, 229, 229, 230, 197, 191, 230, 225, 208, 229]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:23.686535877Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1683,"":"downstairs","":"downstairs","session_id":"b3a7dd17-b723-471c-83ec-ee1f3eec6ab1"} {"msg":"[2]R generation[0..12]: [134, 133, 130, 134, 134, 135, 122, 120, 135, 132, 126, 134]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:23.686546837Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1683,"":"downstairs","":"downstairs","session_id":"b3a7dd17-b723-471c-83ec-ee1f3eec6ab1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:23.686557947Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1683,"":"downstairs","":"downstairs","session_id":"b3a7dd17-b723-471c-83ec-ee1f3eec6ab1"} {"msg":"Max found gen is 136","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:23.686569367Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1683,"":"downstairs","":"downstairs","session_id":"b3a7dd17-b723-471c-83ec-ee1f3eec6ab1"} {"msg":"Generation requested: 136 >= found:136","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:23.686579277Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1683,"":"downstairs","":"downstairs","session_id":"b3a7dd17-b723-471c-83ec-ee1f3eec6ab1"} {"msg":"Next flush: 231","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:23.686588727Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1683,"":"downstairs","":"downstairs","session_id":"b3a7dd17-b723-471c-83ec-ee1f3eec6ab1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:23.686598257Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1683,"":"downstairs","":"downstairs","session_id":"b3a7dd17-b723-471c-83ec-ee1f3eec6ab1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:23.686608177Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1683,"session_id":"b3a7dd17-b723-471c-83ec-ee1f3eec6ab1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:23.686617147Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1683,"client":"0","":"downstairs","session_id":"b3a7dd17-b723-471c-83ec-ee1f3eec6ab1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:23.686626807Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1683,"client":"1","":"downstairs","session_id":"b3a7dd17-b723-471c-83ec-ee1f3eec6ab1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:23.686636547Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1683,"client":"2","":"downstairs","session_id":"b3a7dd17-b723-471c-83ec-ee1f3eec6ab1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:23.686646537Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1683,"session_id":"b3a7dd17-b723-471c-83ec-ee1f3eec6ab1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:23.686655807Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1683,"session_id":"b3a7dd17-b723-471c-83ec-ee1f3eec6ab1"} {"msg":"68191ccf-4ed6-4af7-833b-d754062aa735 is now active with session: b3a7dd17-b723-471c-83ec-ee1f3eec6ab1","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:23.686665937Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1683,"session_id":"b3a7dd17-b723-471c-83ec-ee1f3eec6ab1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:23.686674647Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1683,"session_id":"b3a7dd17-b723-471c-83ec-ee1f3eec6ab1"} {"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:23.686683887Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1683} {"msg":"Activated sub_volume 68191ccf-4ed6-4af7-833b-d754062aa735","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:23.686692117Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1683} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:23.686700467Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1683,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:23.687035508Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1683,"task":"crutest"} 01/30 Write block 580 len 4096 data: 9 02/30 Read block 285 len 8192 03/30 Read block 229 len 32768 04/30 Flush 05/30 Write block 201 len 16384 data: 9 9 11 11 06/30 Write block 271 len 4096 data: 15 07/30 Write block 123 len 28672 data: 8 7 9 8 6 7 6 08/30 Read block 379 len 24576 09/30 Read block 177 len 16384 10/30 Write block 234 len 20480 data: 10 9 10 13 12 11/30 Write block 248 len 16384 data: 11 11 10 12 12/30 Read block 56 len 28672 13/30 Read block 569 len 16384 14/30 Read block 515 len 20480 15/30 Read block 548 len 12288 16/30 Read block 178 len 8192 17/30 Write block 316 len 28672 data: 7 9 10 11 13 13 13 18/30 Read block 286 len 8192 19/30 Write block 354 len 16384 data: 11 11 9 11 20/30 Read block 110 len 16384 21/30 Read block 373 len 32768 22/30 Write block 89 len 36864 data: 16 15 14 13 11 11 11 9 10 23/30 Write block 399 len 4096 data: 9 24/30 Read block 34 len 28672 25/30 Read block 186 len 40960 26/30 Read block 541 len 12288 27/30 Write block 160 len 8192 data: 11 11 28/30 Write block 167 len 40960 data: 9 14 11 11 12 12 12 10 8 7 29/30 Write block 323 len 24576 data: 13 10 11 10 10 11 30/30 Read block 192 len 32768 ---------------------------------------------------------------- Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Read 2 Sent Done Done false 1002 Acked Read 8 Sent Done Done false 1003 Acked Flush 0 Sent Done Done false 1004 Acked Write 4 Sent Done Done false 1005 Acked Write 1 Sent Done Done false 1006 Acked Write 7 Sent Done Done false 1007 Acked Read 6 Sent Done Done false 1008 Acked Read 4 Sent Done Done false 1009 Acked Write 5 Sent Done Done false 1010 Acked Write 4 Sent Done Done false 1011 Acked Read 7 Sent Done Done false 1012 Acked Read 4 Sent Done Done false 1013 Acked Read 5 Sent Done Done false 1014 Acked Read 3 Sent Done Done false 1015 Acked Read 2 Sent Done Done false 1016 Acked Write 7 Sent Done Done false 1017 Acked Read 2 Sent Done Done false 1018 Acked Write 4 Sent Done Done false 1019 Acked Read 4 Sent Done Done false 1020 Acked Read 8 Sent Done Done false 1021 Acked Write 9 Sent Done Done false 1022 Acked Write 1 Sent Done Done false 1023 Acked Read 7 Sent Done Done false 1024 Acked Read 10 Sent Done Done false 1025 Acked Read 3 Sent Done Done false 1026 Acked Write 2 Sent Done Done false 1027 Acked Write 10 Sent Done Sent false 1028 Acked Write 6 Sent Done Sent false 1029 Acked Read 8 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 0 3 32 Done 1 30 27 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1003)) Some(JobId(1003)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:24:23.695544951Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1683} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:23.695727651Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1683,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 134 134 134 228 228 228 F F F 1 020-039 133 133 133 227 227 227 F F F 2 040-059 130 130 130 221 221 221 F F F 3 060-079 134 134 134 229 229 229 F F F 4 080-099 134 134 134 229 229 229 F T T <--- 5 100-119 135 135 135 230 230 230 F F F 6 120-139 122 122 122 197 197 197 F T T <--- 7 140-159 120 120 120 191 191 191 F F F 8 160-179 135 135 135 230 230 230 F T T <--- 9 180-199 132 132 132 225 225 225 F F F 10 200-219 126 126 126 208 208 208 F T T <--- 11 220-239 134 134 134 229 229 229 F T T <--- 12 240-259 134 134 134 229 229 229 F T T <--- 13 260-279 132 132 132 226 226 226 F T T <--- 14 280-299 134 134 134 228 228 228 F F F 15 300-319 134 134 134 228 228 228 F T T <--- 16 320-339 132 132 132 226 226 226 F T T <--- 17 340-359 132 132 132 226 226 226 F T T <--- 18 360-379 125 125 125 203 203 203 F F F 19 380-399 134 134 134 228 228 228 F T T <--- 20 400-419 130 130 130 223 223 223 F F F 21 420-439 129 129 129 217 217 217 F F F 22 440-459 134 134 134 229 229 229 F F F 23 460-479 129 129 129 217 217 217 F F F 24 480-499 134 134 134 228 228 228 F F F 25 500-519 134 134 134 228 228 228 F F F 26 520-539 134 134 134 229 229 229 F F F 27 540-559 129 129 129 217 217 217 F F F 28 560-579 132 132 132 226 226 226 F F F 29 580-599 135 136 136 230 231 231 T F F <--- Max gen: 136, Max flush: 231 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 68 Verifying data now No change after verify Loop: 68 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 134 134 134 228 228 228 F F F 1 020-039 133 133 133 227 227 227 F F F 2 040-059 130 130 130 221 221 221 F F F 3 060-079 134 134 134 229 229 229 F F F 4 080-099 137 137 137 232 232 232 F F F 5 100-119 135 135 135 230 230 230 F F F 6 120-139 137 137 137 232 232 232 F F F 7 140-159 120 120 120 191 191 191 F F F 8 160-179 137 137 137 232 232 232 F F F 9 180-199 132 132 132 225 225 225 F F F 10 200-219 137 137 137 232 232 232 F F F 11 220-239 137 137 137 232 232 232 F F F 12 240-259 137 137 137 232 232 232 F F F 13 260-279 137 137 137 232 232 232 F F F 14 280-299 134 134 134 228 228 228 F F F 15 300-319 137 137 137 232 232 232 F F F 16 320-339 137 137 137 232 232 232 F F F 17 340-359 137 137 137 232 232 232 F F F 18 360-379 125 125 125 203 203 203 F F F 19 380-399 137 137 137 232 232 232 F F F 20 400-419 130 130 130 223 223 223 F F F 21 420-439 129 129 129 217 217 217 F F F 22 440-459 134 134 134 229 229 229 F F F 23 460-479 129 129 129 217 217 217 F F F 24 480-499 134 134 134 228 228 228 F F F 25 500-519 134 134 134 228 228 228 F F F 26 520-539 134 134 134 229 229 229 F F F 27 540-559 129 129 129 217 217 217 F F F 28 560-579 132 132 132 226 226 226 F F F 29 580-599 136 136 136 231 231 231 F F F Max gen: 137, Max flush: 232 Begin loop 69 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:27.954894468Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1691,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:27.96334214Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1691,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:27.963650651Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1691} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:27.963678201Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1691} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:27.963686751Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1691} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:27.963698521Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1691} {"msg":"Crucible f8eaf97e-edcb-4631-9219-87901a346777 has session id: 4c6886af-0fb3-4f73-99af-43085d7c077a","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:27.964053871Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1691,"session_id":"4c6886af-0fb3-4f73-99af-43085d7c077a"} {"msg":"Upstairs opts: Upstairs UUID: f8eaf97e-edcb-4631-9219-87901a346777, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:27.964082372Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1691,"session_id":"4c6886af-0fb3-4f73-99af-43085d7c077a"} {"msg":"Crucible stats registered with UUID: f8eaf97e-edcb-4631-9219-87901a346777","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:27.964093732Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1691,"session_id":"4c6886af-0fb3-4f73-99af-43085d7c077a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:24:27.964102962Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1691,"session_id":"4c6886af-0fb3-4f73-99af-43085d7c077a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:27.964236242Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1691,"":"io task","client":"0","":"downstairs","session_id":"4c6886af-0fb3-4f73-99af-43085d7c077a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:27.964267562Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1691,"":"io task","client":"1","":"downstairs","session_id":"4c6886af-0fb3-4f73-99af-43085d7c077a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:27.964278712Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1691,"":"io task","client":"2","":"downstairs","session_id":"4c6886af-0fb3-4f73-99af-43085d7c077a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:27.964425642Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1691,"task":"crutest"} {"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:27.964444802Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1691} {"msg":"f8eaf97e-edcb-4631-9219-87901a346777 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:27.964460762Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1691,"session_id":"4c6886af-0fb3-4f73-99af-43085d7c077a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:27.964471762Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1691,"client":"0","":"downstairs","session_id":"4c6886af-0fb3-4f73-99af-43085d7c077a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:27.964481622Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1691,"client":"1","":"downstairs","session_id":"4c6886af-0fb3-4f73-99af-43085d7c077a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:27.964491462Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1691,"client":"2","":"downstairs","session_id":"4c6886af-0fb3-4f73-99af-43085d7c077a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:27.964501192Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1691,"":"io task","client":"2","":"downstairs","session_id":"4c6886af-0fb3-4f73-99af-43085d7c077a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:27.964511392Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1691,"":"io task","client":"0","":"downstairs","session_id":"4c6886af-0fb3-4f73-99af-43085d7c077a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:27.964521382Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1691,"":"io task","client":"1","":"downstairs","session_id":"4c6886af-0fb3-4f73-99af-43085d7c077a"} {"msg":"ds_connection connected from Ok(127.0.0.1:43842)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:27.964691512Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1691,"":"io task","client":"2","":"downstairs","session_id":"4c6886af-0fb3-4f73-99af-43085d7c077a"} {"msg":"ds_connection connected from Ok(127.0.0.1:39822)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:27.964742702Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1691,"":"io task","client":"1","":"downstairs","session_id":"4c6886af-0fb3-4f73-99af-43085d7c077a"} {"msg":"ds_connection connected from Ok(127.0.0.1:46027)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:27.964760653Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1691,"":"io task","client":"0","":"downstairs","session_id":"4c6886af-0fb3-4f73-99af-43085d7c077a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:27.965319853Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1691,"client":"0","":"downstairs","session_id":"4c6886af-0fb3-4f73-99af-43085d7c077a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:27.965339033Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1691,"client":"1","":"downstairs","session_id":"4c6886af-0fb3-4f73-99af-43085d7c077a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:27.965413814Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1691,"client":"2","":"downstairs","session_id":"4c6886af-0fb3-4f73-99af-43085d7c077a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:27.965636904Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1691,"session_id":"4c6886af-0fb3-4f73-99af-43085d7c077a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:27.965670414Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1691,"session_id":"4c6886af-0fb3-4f73-99af-43085d7c077a"} {"msg":"[0]R flush_numbers[0..12]: [228, 227, 221, 229, 232, 230, 232, 191, 232, 225, 232, 232]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:27.965767324Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1691,"":"downstairs","":"downstairs","session_id":"4c6886af-0fb3-4f73-99af-43085d7c077a"} {"msg":"[0]R generation[0..12]: [134, 133, 130, 134, 137, 135, 137, 120, 137, 132, 137, 137]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:27.965781104Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1691,"":"downstairs","":"downstairs","session_id":"4c6886af-0fb3-4f73-99af-43085d7c077a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:27.965792194Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1691,"":"downstairs","":"downstairs","session_id":"4c6886af-0fb3-4f73-99af-43085d7c077a"} {"msg":"[1]R flush_numbers[0..12]: [228, 227, 221, 229, 232, 230, 232, 191, 232, 225, 232, 232]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:27.965802954Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1691,"":"downstairs","":"downstairs","session_id":"4c6886af-0fb3-4f73-99af-43085d7c077a"} {"msg":"[1]R generation[0..12]: [134, 133, 130, 134, 137, 135, 137, 120, 137, 132, 137, 137]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:27.965813584Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1691,"":"downstairs","":"downstairs","session_id":"4c6886af-0fb3-4f73-99af-43085d7c077a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:27.965824604Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1691,"":"downstairs","":"downstairs","session_id":"4c6886af-0fb3-4f73-99af-43085d7c077a"} {"msg":"[2]R flush_numbers[0..12]: [228, 227, 221, 229, 232, 230, 232, 191, 232, 225, 232, 232]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:27.965835314Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1691,"":"downstairs","":"downstairs","session_id":"4c6886af-0fb3-4f73-99af-43085d7c077a"} {"msg":"[2]R generation[0..12]: [134, 133, 130, 134, 137, 135, 137, 120, 137, 132, 137, 137]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:27.965845904Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1691,"":"downstairs","":"downstairs","session_id":"4c6886af-0fb3-4f73-99af-43085d7c077a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:27.965856964Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1691,"":"downstairs","":"downstairs","session_id":"4c6886af-0fb3-4f73-99af-43085d7c077a"} {"msg":"Max found gen is 138","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:27.965866364Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1691,"":"downstairs","":"downstairs","session_id":"4c6886af-0fb3-4f73-99af-43085d7c077a"} {"msg":"Generation requested: 138 >= found:138","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:27.965876254Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1691,"":"downstairs","":"downstairs","session_id":"4c6886af-0fb3-4f73-99af-43085d7c077a"} {"msg":"Next flush: 233","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:27.965885724Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1691,"":"downstairs","":"downstairs","session_id":"4c6886af-0fb3-4f73-99af-43085d7c077a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:27.965895164Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1691,"":"downstairs","":"downstairs","session_id":"4c6886af-0fb3-4f73-99af-43085d7c077a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:27.965904824Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1691,"session_id":"4c6886af-0fb3-4f73-99af-43085d7c077a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:27.965913654Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1691,"client":"0","":"downstairs","session_id":"4c6886af-0fb3-4f73-99af-43085d7c077a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:27.965925104Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1691,"client":"1","":"downstairs","session_id":"4c6886af-0fb3-4f73-99af-43085d7c077a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:27.965934814Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1691,"client":"2","":"downstairs","session_id":"4c6886af-0fb3-4f73-99af-43085d7c077a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:27.965944664Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1691,"session_id":"4c6886af-0fb3-4f73-99af-43085d7c077a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:27.965953924Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1691,"session_id":"4c6886af-0fb3-4f73-99af-43085d7c077a"} {"msg":"f8eaf97e-edcb-4631-9219-87901a346777 is now active with session: 4c6886af-0fb3-4f73-99af-43085d7c077a","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:27.965963994Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1691,"session_id":"4c6886af-0fb3-4f73-99af-43085d7c077a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:27.965972814Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1691,"session_id":"4c6886af-0fb3-4f73-99af-43085d7c077a"} {"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:27.965981944Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1691} {"msg":"Activated sub_volume f8eaf97e-edcb-4631-9219-87901a346777","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:27.965989964Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1691} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:27.965998174Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1691,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:27.966338455Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1691,"task":"crutest"} 01/30 Write block 288 len 28672 data: 8 9 10 8 10 13 13 02/30 Write block 149 len 16384 data: 10 10 12 11 03/30 Read block 294 len 24576 04/30 Read block 296 len 36864 05/30 Write block 486 len 20480 data: 13 16 14 15 15 06/30 Read block 365 len 40960 07/30 Read block 438 len 32768 08/30 Read block 479 len 8192 09/30 Write block 41 len 4096 data: 9 10/30 Read block 161 len 8192 11/30 Read block 192 len 8192 12/30 Read block 421 len 12288 13/30 Write block 47 len 36864 data: 6 7 6 10 11 12 11 10 12 14/30 Write block 335 len 16384 data: 10 9 11 9 15/30 Write block 336 len 40960 data: 10 12 10 12 14 15 11 11 12 12 16/30 Write block 51 len 8192 data: 12 13 17/30 Read block 216 len 20480 18/30 Write block 133 len 4096 data: 8 19/30 Flush 20/30 Write block 76 len 40960 data: 9 11 14 15 15 11 11 14 13 15 21/30 Write block 172 len 36864 data: 13 13 11 9 8 10 13 9 8 22/30 Write block 104 len 12288 data: 11 10 12 23/30 Read block 19 len 8192 24/30 Flush 25/30 Write block 409 len 36864 data: 9 10 8 10 11 10 10 12 13 26/30 Write block 59 len 12288 data: 11 11 14 27/30 Write block 185 len 20480 data: 13 13 15 15 12 28/30 Write block 476 len 8192 data: 9 10 29/30 Read block 267 len 40960 30/30 Write block 225 len 24576 data: 7 7 8 10 7 7 ---------------------------------------------------------------- Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Write 4 Done Done Sent false 1002 Acked Read 6 Done Done Sent false 1003 Acked Read 9 Done Done Sent false 1004 Acked Write 5 Done Done Sent false 1005 Acked Read 10 Done Done Sent false 1006 Acked Read 8 Done Done Sent false 1007 Acked Read 2 Done Done Sent false 1008 Acked Write 1 Done Done Sent false 1009 Acked Read 2 Done Done Sent false 1010 Acked Read 2 Done Done Sent false 1011 Acked Read 3 Done Done Sent false 1012 Acked Write 9 Done Done Sent false 1013 Acked Write 4 Done Done Sent false 1014 Acked Write 10 Done Done Sent false 1015 Acked Write 2 Done Done Sent false 1016 Acked Read 5 Done Done Sent false 1017 Acked Write 1 Done Done Sent false 1018 Acked Flush 0 Done Done Sent false 1019 Acked Write 10 Done Done Sent false 1020 Acked Write 9 Done Done Sent false 1021 Acked Write 3 Done Done Sent false 1022 Acked Read 2 Done Done Sent false 1023 Acked Flush 0 Done Done Sent false 1024 Acked Write 9 Done Done Sent false 1025 Acked Write 3 Done Done Sent false 1026 Acked Write 5 Done Done Sent false 1027 Acked Write 2 Done Done Sent false 1028 Acked Read 10 Done Done Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 29 31 Done 29 29 1 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:24:27.974209077Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1691} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:27.974405227Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1691,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 134 134 134 228 228 228 F F F 1 020-039 133 133 133 227 227 227 F F F 2 040-059 138 138 130 233 233 221 T T F <--- 3 060-079 138 138 134 234 234 229 T T F <--- 4 080-099 138 138 137 234 234 232 F F F <--- 5 100-119 138 138 135 234 234 230 F F F <--- 6 120-139 138 138 137 233 233 232 F F F <--- 7 140-159 138 138 120 233 233 191 F F F <--- 8 160-179 138 138 137 234 234 232 F F F <--- 9 180-199 138 138 132 234 234 225 T T F <--- 10 200-219 137 137 137 232 232 232 F F F 11 220-239 137 137 137 232 232 232 T T F <--- 12 240-259 137 137 137 232 232 232 F F F 13 260-279 137 137 137 232 232 232 F F F 14 280-299 138 138 134 233 233 228 F F T <--- 15 300-319 137 137 137 232 232 232 F F F 16 320-339 138 138 137 233 233 232 F F F <--- 17 340-359 138 138 137 233 233 232 F F F <--- 18 360-379 125 125 125 203 203 203 F F F 19 380-399 137 137 137 232 232 232 F F F 20 400-419 130 130 130 223 223 223 T T F <--- 21 420-439 129 129 129 217 217 217 F F F 22 440-459 134 134 134 229 229 229 F F F 23 460-479 129 129 129 217 217 217 T T F <--- 24 480-499 138 138 134 233 233 228 F F F <--- 25 500-519 134 134 134 228 228 228 F F F 26 520-539 134 134 134 229 229 229 F F F 27 540-559 129 129 129 217 217 217 F F F 28 560-579 132 132 132 226 226 226 F F F 29 580-599 136 136 136 231 231 231 F F F Max gen: 138, Max flush: 234 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 69 Verifying data now No change after verify Loop: 69 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 134 134 134 228 228 228 F F F 1 020-039 133 133 133 227 227 227 F F F 2 040-059 139 139 139 235 235 235 F F F 3 060-079 139 139 139 235 235 235 F F F 4 080-099 138 138 138 234 234 234 F F F 5 100-119 138 138 138 234 234 234 F F F 6 120-139 138 138 138 233 233 233 F F F 7 140-159 138 138 138 233 233 233 F F F 8 160-179 138 138 138 234 234 234 F F F 9 180-199 139 139 139 235 235 235 F F F 10 200-219 137 137 137 232 232 232 F F F 11 220-239 139 139 139 235 235 235 F F F 12 240-259 137 137 137 232 232 232 F F F 13 260-279 137 137 137 232 232 232 F F F 14 280-299 138 138 138 233 233 233 F F F 15 300-319 137 137 137 232 232 232 F F F 16 320-339 138 138 138 233 233 233 F F F 17 340-359 138 138 138 233 233 233 F F F 18 360-379 125 125 125 203 203 203 F F F 19 380-399 137 137 137 232 232 232 F F F 20 400-419 139 139 139 235 235 235 F F F 21 420-439 129 129 129 217 217 217 F F F 22 440-459 134 134 134 229 229 229 F F F 23 460-479 139 139 139 235 235 235 F F F 24 480-499 138 138 138 233 233 233 F F F 25 500-519 134 134 134 228 228 228 F F F 26 520-539 134 134 134 229 229 229 F F F 27 540-559 129 129 129 217 217 217 F F F 28 560-579 132 132 132 226 226 226 F F F 29 580-599 136 136 136 231 231 231 F F F Max gen: 139, Max flush: 235 Begin loop 70 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:32.228524917Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1699,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:24:32.235658197Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1699,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:32.235713108Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1699,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:32.242602108Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1699,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:32.242864648Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1699} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:32.242893168Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1699} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:32.242904818Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1699} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:32.242916638Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1699} {"msg":"Crucible 7c1776c5-51a0-4b71-a31f-ca7c62f4b1c1 has session id: a0abff89-47b1-4cb0-b974-4f64ec290bc8","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:32.243257759Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1699,"session_id":"a0abff89-47b1-4cb0-b974-4f64ec290bc8"} {"msg":"Upstairs opts: Upstairs UUID: 7c1776c5-51a0-4b71-a31f-ca7c62f4b1c1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:32.243286569Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1699,"session_id":"a0abff89-47b1-4cb0-b974-4f64ec290bc8"} {"msg":"Crucible stats registered with UUID: 7c1776c5-51a0-4b71-a31f-ca7c62f4b1c1","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:32.243297589Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1699,"session_id":"a0abff89-47b1-4cb0-b974-4f64ec290bc8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:24:32.243306819Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1699,"session_id":"a0abff89-47b1-4cb0-b974-4f64ec290bc8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:32.243451549Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1699,"":"io task","client":"1","":"downstairs","session_id":"a0abff89-47b1-4cb0-b974-4f64ec290bc8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:32.243484249Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1699,"":"io task","client":"0","":"downstairs","session_id":"a0abff89-47b1-4cb0-b974-4f64ec290bc8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:32.243495299Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1699,"":"io task","client":"2","":"downstairs","session_id":"a0abff89-47b1-4cb0-b974-4f64ec290bc8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:32.243627729Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1699,"task":"crutest"} {"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:32.243642889Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1699} {"msg":"7c1776c5-51a0-4b71-a31f-ca7c62f4b1c1 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:32.24368216Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1699,"session_id":"a0abff89-47b1-4cb0-b974-4f64ec290bc8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:32.2436918Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1699,"client":"0","":"downstairs","session_id":"a0abff89-47b1-4cb0-b974-4f64ec290bc8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:32.24370191Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1699,"client":"1","":"downstairs","session_id":"a0abff89-47b1-4cb0-b974-4f64ec290bc8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:32.24371172Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1699,"client":"2","":"downstairs","session_id":"a0abff89-47b1-4cb0-b974-4f64ec290bc8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:32.24372344Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1699,"":"io task","client":"2","":"downstairs","session_id":"a0abff89-47b1-4cb0-b974-4f64ec290bc8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:32.24375658Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1699,"":"io task","client":"0","":"downstairs","session_id":"a0abff89-47b1-4cb0-b974-4f64ec290bc8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:32.24377786Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1699,"":"io task","client":"1","":"downstairs","session_id":"a0abff89-47b1-4cb0-b974-4f64ec290bc8"} {"msg":"ds_connection connected from Ok(127.0.0.1:51197)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:32.24378884Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1699,"":"io task","client":"1","":"downstairs","session_id":"a0abff89-47b1-4cb0-b974-4f64ec290bc8"} {"msg":"ds_connection connected from Ok(127.0.0.1:33397)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:32.24379939Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1699,"":"io task","client":"2","":"downstairs","session_id":"a0abff89-47b1-4cb0-b974-4f64ec290bc8"} {"msg":"ds_connection connected from Ok(127.0.0.1:43590)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:32.24381434Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1699,"":"io task","client":"0","":"downstairs","session_id":"a0abff89-47b1-4cb0-b974-4f64ec290bc8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:32.244544661Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1699,"client":"2","":"downstairs","session_id":"a0abff89-47b1-4cb0-b974-4f64ec290bc8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:32.244609501Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1699,"client":"1","":"downstairs","session_id":"a0abff89-47b1-4cb0-b974-4f64ec290bc8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:32.244664771Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1699,"client":"0","":"downstairs","session_id":"a0abff89-47b1-4cb0-b974-4f64ec290bc8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:32.244875851Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1699,"session_id":"a0abff89-47b1-4cb0-b974-4f64ec290bc8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:32.244918001Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1699,"session_id":"a0abff89-47b1-4cb0-b974-4f64ec290bc8"} {"msg":"[0]R flush_numbers[0..12]: [228, 227, 235, 235, 234, 234, 233, 233, 234, 235, 232, 235]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:32.244997462Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1699,"":"downstairs","":"downstairs","session_id":"a0abff89-47b1-4cb0-b974-4f64ec290bc8"} {"msg":"[0]R generation[0..12]: [134, 133, 139, 139, 138, 138, 138, 138, 138, 139, 137, 139]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:32.245009782Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1699,"":"downstairs","":"downstairs","session_id":"a0abff89-47b1-4cb0-b974-4f64ec290bc8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:32.245021122Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1699,"":"downstairs","":"downstairs","session_id":"a0abff89-47b1-4cb0-b974-4f64ec290bc8"} {"msg":"[1]R flush_numbers[0..12]: [228, 227, 235, 235, 234, 234, 233, 233, 234, 235, 232, 235]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:32.245034542Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1699,"":"downstairs","":"downstairs","session_id":"a0abff89-47b1-4cb0-b974-4f64ec290bc8"} {"msg":"[1]R generation[0..12]: [134, 133, 139, 139, 138, 138, 138, 138, 138, 139, 137, 139]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:32.245045442Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1699,"":"downstairs","":"downstairs","session_id":"a0abff89-47b1-4cb0-b974-4f64ec290bc8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:32.245056652Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1699,"":"downstairs","":"downstairs","session_id":"a0abff89-47b1-4cb0-b974-4f64ec290bc8"} {"msg":"[2]R flush_numbers[0..12]: [228, 227, 235, 235, 234, 234, 233, 233, 234, 235, 232, 235]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:32.245067432Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1699,"":"downstairs","":"downstairs","session_id":"a0abff89-47b1-4cb0-b974-4f64ec290bc8"} {"msg":"[2]R generation[0..12]: [134, 133, 139, 139, 138, 138, 138, 138, 138, 139, 137, 139]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:32.245078142Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1699,"":"downstairs","":"downstairs","session_id":"a0abff89-47b1-4cb0-b974-4f64ec290bc8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:32.245089102Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1699,"":"downstairs","":"downstairs","session_id":"a0abff89-47b1-4cb0-b974-4f64ec290bc8"} {"msg":"Max found gen is 140","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:32.245098612Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1699,"":"downstairs","":"downstairs","session_id":"a0abff89-47b1-4cb0-b974-4f64ec290bc8"} {"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:32.245108542Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1699,"":"downstairs","":"downstairs","session_id":"a0abff89-47b1-4cb0-b974-4f64ec290bc8"} {"msg":"Next flush: 236","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:32.245118062Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1699,"":"downstairs","":"downstairs","session_id":"a0abff89-47b1-4cb0-b974-4f64ec290bc8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:32.245127592Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1699,"":"downstairs","":"downstairs","session_id":"a0abff89-47b1-4cb0-b974-4f64ec290bc8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:32.245137552Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1699,"session_id":"a0abff89-47b1-4cb0-b974-4f64ec290bc8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:32.245146392Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1699,"client":"0","":"downstairs","session_id":"a0abff89-47b1-4cb0-b974-4f64ec290bc8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:32.245156252Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1699,"client":"1","":"downstairs","session_id":"a0abff89-47b1-4cb0-b974-4f64ec290bc8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:32.245165972Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1699,"client":"2","":"downstairs","session_id":"a0abff89-47b1-4cb0-b974-4f64ec290bc8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:32.245175892Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1699,"session_id":"a0abff89-47b1-4cb0-b974-4f64ec290bc8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:32.245186582Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1699,"session_id":"a0abff89-47b1-4cb0-b974-4f64ec290bc8"} {"msg":"7c1776c5-51a0-4b71-a31f-ca7c62f4b1c1 is now active with session: a0abff89-47b1-4cb0-b974-4f64ec290bc8","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:32.245196622Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1699,"session_id":"a0abff89-47b1-4cb0-b974-4f64ec290bc8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:32.245205372Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1699,"session_id":"a0abff89-47b1-4cb0-b974-4f64ec290bc8"} {"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:32.245214622Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1699} {"msg":"Activated sub_volume 7c1776c5-51a0-4b71-a31f-ca7c62f4b1c1","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:32.245222732Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1699} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:32.245230902Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1699,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:32.245540882Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1699,"task":"crutest"} 01/30 Flush 02/30 Write block 428 len 16384 data: 6 6 5 5 03/30 Read block 508 len 28672 04/30 Read block 384 len 12288 05/30 Write block 526 len 12288 data: 15 16 16 06/30 Write block 527 len 16384 data: 17 17 13 10 07/30 Write block 547 len 28672 data: 8 9 9 9 9 11 7 08/30 Write block 362 len 20480 data: 13 15 17 16 16 09/30 Read block 277 len 28672 10/30 Flush 11/30 Write block 470 len 16384 data: 10 10 6 6 12/30 Read block 397 len 40960 13/30 Read block 586 len 24576 14/30 Write block 538 len 28672 data: 10 10 10 10 9 11 13 15/30 Read block 238 len 40960 16/30 Write block 70 len 4096 data: 11 17/30 Read block 269 len 12288 18/30 Write block 179 len 16384 data: 10 9 8 7 19/30 Read block 443 len 32768 20/30 Read block 250 len 4096 21/30 Read block 250 len 16384 22/30 Read block 157 len 36864 23/30 Read block 278 len 16384 24/30 Write block 268 len 32768 data: 16 16 15 16 16 10 12 13 25/30 Read block 232 len 20480 26/30 Write block 423 len 28672 data: 9 9 8 6 6 7 7 27/30 Write block 554 len 16384 data: 6 8 8 11 28/30 Read block 297 len 20480 29/30 Write block 223 len 8192 data: 7 5 30/30 Read block 106 len 12288 ---------------------------------------------------------------- Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 4 Sent Done Done false 1002 Acked Read 7 Sent Done Done false 1003 Acked Read 3 Sent Done Done false 1004 Acked Write 3 Sent Done Done false 1005 Acked Write 4 Sent Done Done false 1006 Acked Write 7 Sent Done Done false 1007 Acked Write 5 Sent Done Done false 1008 Acked Read 7 Sent Done Done false 1009 Acked Flush 0 Sent Done Done false 1010 Acked Write 4 Sent Done Done false 1011 Acked Read 10 Sent Done Done false 1012 Acked Read 6 Sent Done Done false 1013 Acked Write 7 Sent Done Done false 1014 Acked Read 10 Sent Done Done false 1015 Acked Write 1 Sent Done Done false 1016 Acked Read 3 Sent Done Done false 1017 Acked Write 4 Sent Done Done false 1018 Acked Read 8 Sent Done Done false 1019 Acked Read 1 Sent Done Done false 1020 Acked Read 4 Sent Done Done false 1021 Acked Read 9 Sent Done Done false 1022 Acked Read 4 Sent Done Done false 1023 Acked Write 8 Sent Done Done false 1024 Acked Read 5 Sent Done Done false 1025 Acked Write 7 Sent Done Done false 1026 Acked Write 4 Sent Done Done false 1027 Acked Read 5 Sent Done Done false 1028 Acked Write 2 Sent Done Done false 1029 Acked Read 3 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 0 1 30 Done 0 29 28 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1000)) Some(JobId(1009)) Some(JobId(1009)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:24:32.254325916Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1699} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:32.254519746Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1699,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 134 134 134 228 228 228 F F F 1 020-039 133 133 133 227 227 227 F F F 2 040-059 139 139 139 235 235 235 F F F 3 060-079 139 139 139 235 235 235 F T T <--- 4 080-099 138 138 138 234 234 234 F F F 5 100-119 138 138 138 234 234 234 F F F 6 120-139 138 138 138 233 233 233 F F F 7 140-159 138 138 138 233 233 233 F F F 8 160-179 138 138 138 234 234 234 F T T <--- 9 180-199 139 139 139 235 235 235 F T T <--- 10 200-219 137 137 137 232 232 232 F F F 11 220-239 139 139 139 235 235 235 F T T <--- 12 240-259 137 137 137 232 232 232 F F F 13 260-279 137 137 137 232 232 232 F T T <--- 14 280-299 138 138 138 233 233 233 F F F 15 300-319 137 137 137 232 232 232 F F F 16 320-339 138 138 138 233 233 233 F F F 17 340-359 138 138 138 233 233 233 F F F 18 360-379 125 140 140 203 237 237 F F F <--- 19 380-399 137 137 137 232 232 232 F F F 20 400-419 139 139 139 235 235 235 F F F 21 420-439 129 140 140 217 237 237 F T T <--- 22 440-459 134 134 134 229 229 229 F F F 23 460-479 139 139 139 235 235 235 F T T <--- 24 480-499 138 138 138 233 233 233 F F F 25 500-519 134 134 134 228 228 228 F F F 26 520-539 134 140 140 229 237 237 F T T <--- 27 540-559 129 140 140 217 237 237 F T T <--- 28 560-579 132 132 132 226 226 226 F F F 29 580-599 136 136 136 231 231 231 F F F Max gen: 140, Max flush: 237 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 70 Verifying data now No change after verify Loop: 70 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 134 134 134 228 228 228 F F F 1 020-039 133 133 133 227 227 227 F F F 2 040-059 139 139 139 235 235 235 F F F 3 060-079 141 141 141 238 238 238 F F F 4 080-099 138 138 138 234 234 234 F F F 5 100-119 138 138 138 234 234 234 F F F 6 120-139 138 138 138 233 233 233 F F F 7 140-159 138 138 138 233 233 233 F F F 8 160-179 141 141 141 238 238 238 F F F 9 180-199 141 141 141 238 238 238 F F F 10 200-219 137 137 137 232 232 232 F F F 11 220-239 141 141 141 238 238 238 F F F 12 240-259 137 137 137 232 232 232 F F F 13 260-279 141 141 141 238 238 238 F F F 14 280-299 138 138 138 233 233 233 F F F 15 300-319 137 137 137 232 232 232 F F F 16 320-339 138 138 138 233 233 233 F F F 17 340-359 138 138 138 233 233 233 F F F 18 360-379 140 140 140 237 237 237 F F F 19 380-399 137 137 137 232 232 232 F F F 20 400-419 139 139 139 235 235 235 F F F 21 420-439 141 141 141 238 238 238 F F F 22 440-459 134 134 134 229 229 229 F F F 23 460-479 141 141 141 238 238 238 F F F 24 480-499 138 138 138 233 233 233 F F F 25 500-519 134 134 134 228 228 228 F F F 26 520-539 141 141 141 238 238 238 F F F 27 540-559 141 141 141 238 238 238 F F F 28 560-579 132 132 132 226 226 226 F F F 29 580-599 136 136 136 231 231 231 F F F Max gen: 141, Max flush: 238 Begin loop 71 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:36.510374038Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1707,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:36.518672721Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1707,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:36.518962091Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1707} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:36.518986751Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1707} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:36.518995481Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1707} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:36.519007291Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1707} {"msg":"Crucible 0fc2ae29-f859-418d-9080-24ce4def7844 has session id: 656ebfeb-cef2-4368-801f-24b412986a16","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:36.519338212Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1707,"session_id":"656ebfeb-cef2-4368-801f-24b412986a16"} {"msg":"Upstairs opts: Upstairs UUID: 0fc2ae29-f859-418d-9080-24ce4def7844, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:36.519367932Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1707,"session_id":"656ebfeb-cef2-4368-801f-24b412986a16"} {"msg":"Crucible stats registered with UUID: 0fc2ae29-f859-418d-9080-24ce4def7844","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:36.519381272Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1707,"session_id":"656ebfeb-cef2-4368-801f-24b412986a16"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:24:36.519390612Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1707,"session_id":"656ebfeb-cef2-4368-801f-24b412986a16"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:36.519517792Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1707,"":"io task","client":"0","":"downstairs","session_id":"656ebfeb-cef2-4368-801f-24b412986a16"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:36.519535762Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1707,"":"io task","client":"2","":"downstairs","session_id":"656ebfeb-cef2-4368-801f-24b412986a16"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:36.519545882Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1707,"":"io task","client":"1","":"downstairs","session_id":"656ebfeb-cef2-4368-801f-24b412986a16"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:36.519690452Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1707,"task":"crutest"} {"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:36.519707132Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1707} {"msg":"0fc2ae29-f859-418d-9080-24ce4def7844 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:36.519724662Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1707,"session_id":"656ebfeb-cef2-4368-801f-24b412986a16"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:36.519733532Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1707,"client":"0","":"downstairs","session_id":"656ebfeb-cef2-4368-801f-24b412986a16"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:36.519743242Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1707,"client":"1","":"downstairs","session_id":"656ebfeb-cef2-4368-801f-24b412986a16"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:36.519752982Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1707,"client":"2","":"downstairs","session_id":"656ebfeb-cef2-4368-801f-24b412986a16"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:36.519762432Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1707,"":"io task","client":"2","":"downstairs","session_id":"656ebfeb-cef2-4368-801f-24b412986a16"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:36.519772422Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1707,"":"io task","client":"0","":"downstairs","session_id":"656ebfeb-cef2-4368-801f-24b412986a16"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:36.519782482Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1707,"":"io task","client":"1","":"downstairs","session_id":"656ebfeb-cef2-4368-801f-24b412986a16"} {"msg":"ds_connection connected from Ok(127.0.0.1:53178)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:36.519886522Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1707,"":"io task","client":"2","":"downstairs","session_id":"656ebfeb-cef2-4368-801f-24b412986a16"} {"msg":"ds_connection connected from Ok(127.0.0.1:37442)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:36.519900893Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1707,"":"io task","client":"0","":"downstairs","session_id":"656ebfeb-cef2-4368-801f-24b412986a16"} {"msg":"ds_connection connected from Ok(127.0.0.1:51536)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:36.519997153Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1707,"":"io task","client":"1","":"downstairs","session_id":"656ebfeb-cef2-4368-801f-24b412986a16"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:36.520603434Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1707,"client":"0","":"downstairs","session_id":"656ebfeb-cef2-4368-801f-24b412986a16"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:36.520616864Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1707,"client":"1","":"downstairs","session_id":"656ebfeb-cef2-4368-801f-24b412986a16"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:36.520630784Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1707,"client":"2","":"downstairs","session_id":"656ebfeb-cef2-4368-801f-24b412986a16"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:36.520936914Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1707,"session_id":"656ebfeb-cef2-4368-801f-24b412986a16"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:36.520946684Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1707,"session_id":"656ebfeb-cef2-4368-801f-24b412986a16"} {"msg":"[0]R flush_numbers[0..12]: [228, 227, 235, 238, 234, 234, 233, 233, 238, 238, 232, 238]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:36.521023434Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1707,"":"downstairs","":"downstairs","session_id":"656ebfeb-cef2-4368-801f-24b412986a16"} {"msg":"[0]R generation[0..12]: [134, 133, 139, 141, 138, 138, 138, 138, 141, 141, 137, 141]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:36.521034724Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1707,"":"downstairs","":"downstairs","session_id":"656ebfeb-cef2-4368-801f-24b412986a16"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:36.521045854Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1707,"":"downstairs","":"downstairs","session_id":"656ebfeb-cef2-4368-801f-24b412986a16"} {"msg":"[1]R flush_numbers[0..12]: [228, 227, 235, 238, 234, 234, 233, 233, 238, 238, 232, 238]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:36.521056684Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1707,"":"downstairs","":"downstairs","session_id":"656ebfeb-cef2-4368-801f-24b412986a16"} {"msg":"[1]R generation[0..12]: [134, 133, 139, 141, 138, 138, 138, 138, 141, 141, 137, 141]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:36.521067354Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1707,"":"downstairs","":"downstairs","session_id":"656ebfeb-cef2-4368-801f-24b412986a16"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:36.521078434Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1707,"":"downstairs","":"downstairs","session_id":"656ebfeb-cef2-4368-801f-24b412986a16"} {"msg":"[2]R flush_numbers[0..12]: [228, 227, 235, 238, 234, 234, 233, 233, 238, 238, 232, 238]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:36.521089174Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1707,"":"downstairs","":"downstairs","session_id":"656ebfeb-cef2-4368-801f-24b412986a16"} {"msg":"[2]R generation[0..12]: [134, 133, 139, 141, 138, 138, 138, 138, 141, 141, 137, 141]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:36.521099954Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1707,"":"downstairs","":"downstairs","session_id":"656ebfeb-cef2-4368-801f-24b412986a16"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:36.521113114Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1707,"":"downstairs","":"downstairs","session_id":"656ebfeb-cef2-4368-801f-24b412986a16"} {"msg":"Max found gen is 142","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:36.521122664Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1707,"":"downstairs","":"downstairs","session_id":"656ebfeb-cef2-4368-801f-24b412986a16"} {"msg":"Generation requested: 142 >= found:142","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:36.521132554Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1707,"":"downstairs","":"downstairs","session_id":"656ebfeb-cef2-4368-801f-24b412986a16"} {"msg":"Next flush: 239","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:36.521141954Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1707,"":"downstairs","":"downstairs","session_id":"656ebfeb-cef2-4368-801f-24b412986a16"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:36.521151434Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1707,"":"downstairs","":"downstairs","session_id":"656ebfeb-cef2-4368-801f-24b412986a16"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:36.521161234Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1707,"session_id":"656ebfeb-cef2-4368-801f-24b412986a16"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:36.521170044Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1707,"client":"0","":"downstairs","session_id":"656ebfeb-cef2-4368-801f-24b412986a16"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:36.521179774Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1707,"client":"1","":"downstairs","session_id":"656ebfeb-cef2-4368-801f-24b412986a16"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:36.521189384Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1707,"client":"2","":"downstairs","session_id":"656ebfeb-cef2-4368-801f-24b412986a16"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:36.521199394Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1707,"session_id":"656ebfeb-cef2-4368-801f-24b412986a16"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:36.521208614Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1707,"session_id":"656ebfeb-cef2-4368-801f-24b412986a16"} {"msg":"0fc2ae29-f859-418d-9080-24ce4def7844 is now active with session: 656ebfeb-cef2-4368-801f-24b412986a16","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:36.521218784Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1707,"session_id":"656ebfeb-cef2-4368-801f-24b412986a16"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:36.521227665Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1707,"session_id":"656ebfeb-cef2-4368-801f-24b412986a16"} {"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:36.521237065Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1707} {"msg":"Activated sub_volume 0fc2ae29-f859-418d-9080-24ce4def7844","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:36.521245175Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1707} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:36.521253365Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1707Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:36.521519605Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1707,"task":"crutest"} 01/30 Write block 488 len 4096 data: 15 02/30 Write block 131 len 20480 data: 7 9 9 8 11 03/30 Write block 30 len 28672 data: 14 14 11 9 7 5 6 04/30 Read block 427 len 40960 05/30 Write block 152 len 36864 data: 12 10 13 12 12 11 14 12 12 06/30 Write block 170 len 24576 data: 12 13 14 14 12 10 07/30 Read block 290 len 12288 08/30 Read block 391 len 24576 09/30 Write block 474 len 36864 data: 7 7 10 11 11 8 8 7 7 10/30 Write block 338 len 28672 data: 11 13 15 16 12 12 13 11/30 Read block 548 len 36864 12/30 Read block 440 len 12288 13/30 Write block 403 len 40960 data: 5 7 7 6 9 10 10 11 9 11 14/30 Flush 15/30 Flush 16/30 Write block 193 len 12288 data: 13 9 10 17/30 Read block 71 len 8192 18/30 Write block 492 len 20480 data: 12 14 13 13 11 19/30 Write block 202 len 8192 data: 10 12 20/30 Write block 271 len 20480 data: 17 17 11 13 14 21/30 Write block 563 len 36864 data: 7 6 9 10 13 10 12 13 14 22/30 Write block 19 len 28672 data: 13 14 15 14 15 14 13 23/30 Read block 438 len 24576 24/30 Read block 23 len 12288 25/30 Read block 121 len 32768 26/30 Read block 145 len 32768 27/30 Read block 138 len 4096 28/30 Read block 197 len 20480 29/30 Write block 283 len 16384 data: 8 6 8 9 30/30 Read block 422 len 16384 ---------------------------------------------------------------- Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Write 5 Done Done Done false 1002 Acked Write 7 Done Done Done false 1003 Acked Read 10 Done Done Done false 1004 Acked Write 9 Done Done Done false 1005 Acked Write 6 Done Done Done false 1006 Acked Read 3 Done Done Done false 1007 Acked Read 6 Done Done Done false 1008 Acked Write 9 Done Done Done false 1009 Acked Write 7 Done Done Done false 1010 Acked Read 9 Done Sent Done false 1011 Acked Read 3 Done Sent Done false 1012 Acked Write 10 Done Sent Done false 1013 Acked Flush 0 Done Sent Done false 1014 Acked Flush 0 Done Sent Done false 1015 Acked Write 3 Done Sent Done false 1016 Acked Read 2 Done Sent Done false 1017 Acked Write 5 Done Sent Done false 1018 Acked Write 2 Done Sent Done false 1019 Acked Write 5 Done Sent Done false 1020 Acked Write 9 Done Sent Done false 1021 Acked Write 7 Done Sent Done false 1022 Acked Read 6 Done Sent Done false 1023 Acked Read 3 Done Sent Done false 1024 Acked Read 8 Done Sent Done false 1025 Acked Read 8 Done Sent Done false 1026 Acked Read 1 Done Sent Done false 1027 Acked Read 5 Done Sent Done false 1028 Acked Write 4 Done Sent Sent false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 20 2 22 Done 30 10 28 68 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) None Some(JobId(1014)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:24:36.529996928Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1707} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:36.530209808Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1707,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 134 134 134 228 228 228 T F T <--- 1 020-039 142 133 142 239 227 239 T T T <--- 2 040-059 139 139 139 235 235 235 F F F 3 060-079 141 141 141 238 238 238 F F F 4 080-099 138 138 138 234 234 234 F F F 5 100-119 138 138 138 234 234 234 F F F 6 120-139 142 138 142 239 233 239 F T F <--- 7 140-159 142 138 142 239 233 239 F T F <--- 8 160-179 142 141 142 239 238 239 F T F <--- 9 180-199 141 141 141 238 238 238 T F T <--- 10 200-219 137 137 137 232 232 232 T F T <--- 11 220-239 141 141 141 238 238 238 F F F 12 240-259 137 137 137 232 232 232 F F F 13 260-279 141 141 141 238 238 238 T F T <--- 14 280-299 138 138 138 233 233 233 T F T <--- 15 300-319 137 137 137 232 232 232 F F F 16 320-339 142 138 142 239 233 239 F T F <--- 17 340-359 142 138 142 239 233 239 F T F <--- 18 360-379 140 140 140 237 237 237 F F F 19 380-399 137 137 137 232 232 232 F F F 20 400-419 142 139 142 239 235 239 F F F <--- 21 420-439 141 141 141 238 238 238 F F F 22 440-459 134 134 134 229 229 229 F F F 23 460-479 142 141 142 239 238 239 F T F <--- 24 480-499 142 138 142 239 233 239 T T T <--- 25 500-519 134 134 134 228 228 228 F F F 26 520-539 141 141 141 238 238 238 F F F 27 540-559 141 141 141 238 238 238 F F F 28 560-579 132 132 132 226 226 226 T F T <--- 29 580-599 136 136 136 231 231 231 F F F Max gen: 142, Max flush: 239 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 71 Verifying data now No change after verify Loop: 71 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 143 143 143 240 240 240 F F F 1 020-039 143 143 143 240 240 240 F F F 2 040-059 139 139 139 235 235 235 F F F 3 060-079 141 141 141 238 238 238 F F F 4 080-099 138 138 138 234 234 234 F F F 5 100-119 138 138 138 234 234 234 F F F 6 120-139 142 142 142 239 239 239 F F F 7 140-159 142 142 142 239 239 239 F F F 8 160-179 142 142 142 239 239 239 F F F 9 180-199 143 143 143 240 240 240 F F F 10 200-219 143 143 143 240 240 240 F F F 11 220-239 141 141 141 238 238 238 F F F 12 240-259 137 137 137 232 232 232 F F F 13 260-279 143 143 143 240 240 240 F F F 14 280-299 143 143 143 240 240 240 F F F 15 300-319 137 137 137 232 232 232 F F F 16 320-339 142 142 142 239 239 239 F F F 17 340-359 142 142 142 239 239 239 F F F 18 360-379 140 140 140 237 237 237 F F F 19 380-399 137 137 137 232 232 232 F F F 20 400-419 142 142 142 239 239 239 F F F 21 420-439 141 141 141 238 238 238 F F F 22 440-459 134 134 134 229 229 229 F F F 23 460-479 142 142 142 239 239 239 F F F 24 480-499 143 143 143 240 240 240 F F F 25 500-519 134 134 134 228 228 228 F F F 26 520-539 141 141 141 238 238 238 F F F 27 540-559 141 141 141 238 238 238 F F F 28 560-579 143 143 143 240 240 240 F F F 29 580-599 136 136 136 231 231 231 F F F Max gen: 143, Max flush: 240 Begin loop 72 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:40.79273956Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1715,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:40.800861753Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1715,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:40.801156943Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1715} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:40.801184893Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1715} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:40.801193433Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1715} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:40.801205523Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1715} {"msg":"Crucible 4f4bd737-7aa9-47bc-8e46-b5ffe65d39a6 has session id: a6dd334e-a10e-4988-9ae4-bbfff852d825","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:40.801541424Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1715,"session_id":"a6dd334e-a10e-4988-9ae4-bbfff852d825"} {"msg":"Upstairs opts: Upstairs UUID: 4f4bd737-7aa9-47bc-8e46-b5ffe65d39a6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:40.801571024Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1715,"session_id":"a6dd334e-a10e-4988-9ae4-bbfff852d825"} {"msg":"Crucible stats registered with UUID: 4f4bd737-7aa9-47bc-8e46-b5ffe65d39a6","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:40.801582534Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1715,"session_id":"a6dd334e-a10e-4988-9ae4-bbfff852d825"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:24:40.801591844Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1715,"session_id":"a6dd334e-a10e-4988-9ae4-bbfff852d825"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:40.801719944Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1715,"":"io task","client":"2","":"downstairs","session_id":"a6dd334e-a10e-4988-9ae4-bbfff852d825"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:40.801737304Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1715,"":"io task","client":"0","":"downstairs","session_id":"a6dd334e-a10e-4988-9ae4-bbfff852d825"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:40.801747524Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1715,"":"io task","client":"1","":"downstairs","session_id":"a6dd334e-a10e-4988-9ae4-bbfff852d825"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:40.801917164Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1715,"task":"crutest"} {"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:40.801935654Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1715} {"msg":"4f4bd737-7aa9-47bc-8e46-b5ffe65d39a6 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:40.801961254Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1715,"session_id":"a6dd334e-a10e-4988-9ae4-bbfff852d825"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:40.801970664Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1715,"client":"0","":"downstairs","session_id":"a6dd334e-a10e-4988-9ae4-bbfff852d825"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:40.801980504Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1715,"client":"1","":"downstairs","session_id":"a6dd334e-a10e-4988-9ae4-bbfff852d825"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:40.801990214Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1715,"client":"2","":"downstairs","session_id":"a6dd334e-a10e-4988-9ae4-bbfff852d825"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:40.802014764Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1715,"":"io task","client":"2","":"downstairs","session_id":"a6dd334e-a10e-4988-9ae4-bbfff852d825"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:40.802024854Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1715,"":"io task","client":"0","":"downstairs","session_id":"a6dd334e-a10e-4988-9ae4-bbfff852d825"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:40.802035024Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1715,"":"io task","client":"1","":"downstairs","session_id":"a6dd334e-a10e-4988-9ae4-bbfff852d825"} {"msg":"ds_connection connected from Ok(127.0.0.1:64724)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:40.802091774Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1715,"":"io task","client":"1","":"downstairs","session_id":"a6dd334e-a10e-4988-9ae4-bbfff852d825"} {"msg":"ds_connection connected from Ok(127.0.0.1:46374)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:40.802175665Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1715,"":"io task","client":"2","":"downstairs","session_id":"a6dd334e-a10e-4988-9ae4-bbfff852d825"} {"msg":"ds_connection connected from Ok(127.0.0.1:45169)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:40.802187485Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1715,"":"io task","client":"0","":"downstairs","session_id":"a6dd334e-a10e-4988-9ae4-bbfff852d825"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:40.802715955Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1715,"client":"0","":"downstairs","session_id":"a6dd334e-a10e-4988-9ae4-bbfff852d825"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:40.802728785Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1715,"client":"1","":"downstairs","session_id":"a6dd334e-a10e-4988-9ae4-bbfff852d825"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:40.802868236Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1715,"client":"2","":"downstairs","session_id":"a6dd334e-a10e-4988-9ae4-bbfff852d825"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:40.803040806Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1715,"session_id":"a6dd334e-a10e-4988-9ae4-bbfff852d825"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:40.803052966Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1715,"session_id":"a6dd334e-a10e-4988-9ae4-bbfff852d825"} {"msg":"[0]R flush_numbers[0..12]: [240, 240, 235, 238, 234, 234, 239, 239, 239, 240, 240, 238]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:40.803173896Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1715,"":"downstairs","":"downstairs","session_id":"a6dd334e-a10e-4988-9ae4-bbfff852d825"} {"msg":"[0]R generation[0..12]: [143, 143, 139, 141, 138, 138, 142, 142, 142, 143, 143, 141]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:40.803185216Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1715,"":"downstairs","":"downstairs","session_id":"a6dd334e-a10e-4988-9ae4-bbfff852d825"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:40.803196226Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1715,"":"downstairs","":"downstairs","session_id":"a6dd334e-a10e-4988-9ae4-bbfff852d825"} {"msg":"[1]R flush_numbers[0..12]: [240, 240, 235, 238, 234, 234, 239, 239, 239, 240, 240, 238]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:40.803206956Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1715,"":"downstairs","":"downstairs","session_id":"a6dd334e-a10e-4988-9ae4-bbfff852d825"} {"msg":"[1]R generation[0..12]: [143, 143, 139, 141, 138, 138, 142, 142, 142, 143, 143, 141]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:40.803217746Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1715,"":"downstairs","":"downstairs","session_id":"a6dd334e-a10e-4988-9ae4-bbfff852d825"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:40.803228916Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1715,"":"downstairs","":"downstairs","session_id":"a6dd334e-a10e-4988-9ae4-bbfff852d825"} {"msg":"[2]R flush_numbers[0..12]: [240, 240, 235, 238, 234, 234, 239, 239, 239, 240, 240, 238]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:40.803239696Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1715,"":"downstairs","":"downstairs","session_id":"a6dd334e-a10e-4988-9ae4-bbfff852d825"} {"msg":"[2]R generation[0..12]: [143, 143, 139, 141, 138, 138, 142, 142, 142, 143, 143, 141]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:40.803250406Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1715,"":"downstairs","":"downstairs","session_id":"a6dd334e-a10e-4988-9ae4-bbfff852d825"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:40.803261496Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1715,"":"downstairs","":"downstairs","session_id":"a6dd334e-a10e-4988-9ae4-bbfff852d825"} {"msg":"Max found gen is 144","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:40.803271126Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1715,"":"downstairs","":"downstairs","session_id":"a6dd334e-a10e-4988-9ae4-bbfff852d825"} {"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:40.803280976Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1715,"":"downstairs","":"downstairs","session_id":"a6dd334e-a10e-4988-9ae4-bbfff852d825"} {"msg":"Next flush: 241","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:40.803290356Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1715,"":"downstairs","":"downstairs","session_id":"a6dd334e-a10e-4988-9ae4-bbfff852d825"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:40.803299896Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1715,"":"downstairs","":"downstairs","session_id":"a6dd334e-a10e-4988-9ae4-bbfff852d825"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:40.803310946Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1715,"session_id":"a6dd334e-a10e-4988-9ae4-bbfff852d825"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:40.803319826Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1715,"client":"0","":"downstairs","session_id":"a6dd334e-a10e-4988-9ae4-bbfff852d825"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:40.803329646Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1715,"client":"1","":"downstairs","session_id":"a6dd334e-a10e-4988-9ae4-bbfff852d825"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:40.803339316Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1715,"client":"2","":"downstairs","session_id":"a6dd334e-a10e-4988-9ae4-bbfff852d825"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:40.803349346Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1715,"session_id":"a6dd334e-a10e-4988-9ae4-bbfff852d825"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:40.803358626Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1715,"session_id":"a6dd334e-a10e-4988-9ae4-bbfff852d825"} {"msg":"4f4bd737-7aa9-47bc-8e46-b5ffe65d39a6 is now active with session: a6dd334e-a10e-4988-9ae4-bbfff852d825","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:40.803368736Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1715,"session_id":"a6dd334e-a10e-4988-9ae4-bbfff852d825"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:40.803377506Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1715,"session_id":"a6dd334e-a10e-4988-9ae4-bbfff852d825"} {"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:40.803386666Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1715} {"msg":"Activated sub_volume 4f4bd737-7aa9-47bc-8e46-b5ffe65d39a6","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:40.803394696Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1715} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:40.803402876Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1715,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:40.803712227Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1715,"task":"crutest"} 01/30 Write block 190 len 36864 data: 11 13 14 14 10 11 10 11 8 02/30 Write block 543 len 24576 data: 12 14 10 10 9 10 03/30 Write block 483 len 32768 data: 8 10 12 14 17 16 16 16 04/30 Read block 283 len 20480 05/30 Read block 266 len 32768 06/30 Read block 270 len 40960 07/30 Read block 499 len 28672 08/30 Read block 201 len 12288 09/30 Read block 217 len 28672 10/30 Read block 451 len 20480 11/30 Read block 386 len 16384 12/30 Flush 13/30 Write block 47 len 16384 data: 7 8 7 11 14/30 Write block 179 len 12288 data: 11 10 9 15/30 Write block 106 len 16384 data: 13 11 11 10 16/30 Read block 250 len 28672 17/30 Write block 368 len 40960 data: 13 10 9 11 13 10 11 13 15 13 18/30 Read block 466 len 24576 19/30 Write block 412 len 16384 data: 12 12 11 11 20/30 Write block 499 len 20480 data: 13 10 10 13 11 21/30 Read block 247 len 16384 22/30 Write block 122 len 4096 data: 8 23/30 Flush 24/30 Write block 456 len 4096 data: 14 25/30 Read block 318 len 36864 26/30 Read block 574 len 40960 27/30 Write block 346 len 20480 data: 12 14 12 11 8 28/30 Read block 253 len 40960 29/30 Read block 205 len 12288 30/30 Write block 562 len 36864 data: 7 8 7 10 11 14 11 13 14 ---------------------------------------------------------------- Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Write 6 Done Done Sent false 1002 Acked Write 8 Done Done Sent false 1003 Acked Read 5 Done Done Sent false 1004 Acked Read 8 Done Done Sent false 1005 Acked Read 10 Done Done Sent false 1006 Acked Read 7 Done Done Sent false 1007 Acked Read 3 Done Done Sent false 1008 Acked Read 7 Done Done Sent false 1009 Acked Read 5 Done Done Sent false 1010 Acked Read 4 Done Done Sent false 1011 Acked Flush 0 Done Done Sent false 1012 Acked Write 4 Done Done Sent false 1013 Acked Write 3 Done Done Sent false 1014 Acked Write 4 Done Done Sent false 1015 Acked Read 7 Done Done Sent false 1016 Acked Write 10 Done Done Sent false 1017 Acked Read 6 Done Done Sent false 1018 Acked Write 4 Done Done Sent false 1019 Acked Write 5 Done Done Sent false 1020 Acked Read 4 Done Done Sent false 1021 Acked Write 1 Done Done Sent false 1022 Acked Flush 0 Done Done Sent false 1023 Acked Write 1 Done Done Sent false 1024 Acked Read 9 Done Done Sent false 1025 Acked Read 10 Done Done Sent false 1026 Acked Write 5 Done Done Sent false 1027 Acked Read 10 Done Done Sent false 1028 Acked Read 3 Done Done Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 29 31 Done 29 29 1 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:24:40.81271137Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1715} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:40.812896191Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1715,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 143 143 143 240 240 240 F F F 1 020-039 143 143 143 240 240 240 F F F 2 040-059 144 144 139 242 242 235 F F F <--- 3 060-079 141 141 141 238 238 238 F F F 4 080-099 138 138 138 234 234 234 F F F 5 100-119 144 144 138 242 242 234 F F F <--- 6 120-139 144 144 142 242 242 239 F F F <--- 7 140-159 142 142 142 239 239 239 F F F 8 160-179 144 144 142 242 242 239 F F F <--- 9 180-199 144 144 143 242 242 240 F F T <--- 10 200-219 143 143 143 240 240 240 F F F 11 220-239 141 141 141 238 238 238 F F F 12 240-259 137 137 137 232 232 232 F F F 13 260-279 143 143 143 240 240 240 F F F 14 280-299 143 143 143 240 240 240 F F F 15 300-319 137 137 137 232 232 232 F F F 16 320-339 142 142 142 239 239 239 F F F 17 340-359 142 142 142 239 239 239 T T F <--- 18 360-379 144 144 140 242 242 237 F F F <--- 19 380-399 137 137 137 232 232 232 F F F 20 400-419 144 144 142 242 242 239 F F F <--- 21 420-439 141 141 141 238 238 238 F F F 22 440-459 134 134 134 229 229 229 T T F <--- 23 460-479 142 142 142 239 239 239 F F F 24 480-499 144 144 143 242 242 240 F F F <--- 25 500-519 144 144 134 242 242 228 F F F <--- 26 520-539 141 141 141 238 238 238 F F F 27 540-559 144 144 141 241 241 238 F F F <--- 28 560-579 143 143 143 240 240 240 T T F <--- 29 580-599 136 136 136 231 231 231 F F F Max gen: 144, Max flush: 242 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 72 Verifying data now No change after verify Loop: 72 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 143 143 143 240 240 240 F F F 1 020-039 143 143 143 240 240 240 F F F 2 040-059 144 144 144 242 242 242 F F F 3 060-079 141 141 141 238 238 238 F F F 4 080-099 138 138 138 234 234 234 F F F 5 100-119 144 144 144 242 242 242 F F F 6 120-139 144 144 144 242 242 242 F F F 7 140-159 142 142 142 239 239 239 F F F 8 160-179 144 144 144 242 242 242 F F F 9 180-199 144 144 144 242 242 242 F F F 10 200-219 143 143 143 240 240 240 F F F 11 220-239 141 141 141 238 238 238 F F F 12 240-259 137 137 137 232 232 232 F F F 13 260-279 143 143 143 240 240 240 F F F 14 280-299 143 143 143 240 240 240 F F F 15 300-319 137 137 137 232 232 232 F F F 16 320-339 142 142 142 239 239 239 F F F 17 340-359 145 145 145 243 243 243 F F F 18 360-379 144 144 144 242 242 242 F F F 19 380-399 137 137 137 232 232 232 F F F 20 400-419 144 144 144 242 242 242 F F F 21 420-439 141 141 141 238 238 238 F F F 22 440-459 145 145 145 243 243 243 F F F 23 460-479 142 142 142 239 239 239 F F F 24 480-499 144 144 144 242 242 242 F F F 25 500-519 144 144 144 242 242 242 F F F 26 520-539 141 141 141 238 238 238 F F F 27 540-559 144 144 144 241 241 241 F F F 28 560-579 145 145 145 243 243 243 F F F 29 580-599 136 136 136 231 231 231 F F F Max gen: 145, Max flush: 243 Begin loop 73 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:45.069174213Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1723,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:45.077314235Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1723,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:45.077620726Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1723} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:45.077650126Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1723} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:45.077662636Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1723} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:45.077674766Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1723} {"msg":"Crucible d78f172e-d6f6-462d-b161-b956d46bef8c has session id: 87ce64a4-b362-4c5a-9fd8-10964fa38314","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:45.078001116Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1723,"session_id":"87ce64a4-b362-4c5a-9fd8-10964fa38314"} {"msg":"Upstairs opts: Upstairs UUID: d78f172e-d6f6-462d-b161-b956d46bef8c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:45.078030867Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1723,"session_id":"87ce64a4-b362-4c5a-9fd8-10964fa38314"} {"msg":"Crucible stats registered with UUID: d78f172e-d6f6-462d-b161-b956d46bef8c","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:45.078042077Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1723,"session_id":"87ce64a4-b362-4c5a-9fd8-10964fa38314"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:24:45.078051367Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1723,"session_id":"87ce64a4-b362-4c5a-9fd8-10964fa38314"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:45.078176117Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1723,"":"io task","client":"2","":"downstairs","session_id":"87ce64a4-b362-4c5a-9fd8-10964fa38314"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:45.078196227Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1723,"":"io task","client":"0","":"downstairs","session_id":"87ce64a4-b362-4c5a-9fd8-10964fa38314"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:45.078206357Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1723,"":"io task","client":"1","":"downstairs","session_id":"87ce64a4-b362-4c5a-9fd8-10964fa38314"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:45.078337907Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1723,"task":"crutest"} {"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:45.078357147Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1723} {"msg":"d78f172e-d6f6-462d-b161-b956d46bef8c active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:45.078384437Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1723,"session_id":"87ce64a4-b362-4c5a-9fd8-10964fa38314"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:45.078393887Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1723,"client":"0","":"downstairs","session_id":"87ce64a4-b362-4c5a-9fd8-10964fa38314"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:45.078403537Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1723,"client":"1","":"downstairs","session_id":"87ce64a4-b362-4c5a-9fd8-10964fa38314"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:45.078413287Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1723,"client":"2","":"downstairs","session_id":"87ce64a4-b362-4c5a-9fd8-10964fa38314"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:45.078424827Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1723,"":"io task","client":"2","":"downstairs","session_id":"87ce64a4-b362-4c5a-9fd8-10964fa38314"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:45.078434857Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1723,"":"io task","client":"0","":"downstairs","session_id":"87ce64a4-b362-4c5a-9fd8-10964fa38314"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:45.078444977Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1723,"":"io task","client":"1","":"downstairs","session_id":"87ce64a4-b362-4c5a-9fd8-10964fa38314"} {"msg":"ds_connection connected from Ok(127.0.0.1:36648)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:45.078542977Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1723,"":"io task","client":"0","":"downstairs","session_id":"87ce64a4-b362-4c5a-9fd8-10964fa38314"} {"msg":"ds_connection connected from Ok(127.0.0.1:51478)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:45.078567717Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1723,"":"io task","client":"1","":"downstairs","session_id":"87ce64a4-b362-4c5a-9fd8-10964fa38314"} {"msg":"ds_connection connected from Ok(127.0.0.1:58741)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:45.078579127Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1723,"":"io task","client":"2","":"downstairs","session_id":"87ce64a4-b362-4c5a-9fd8-10964fa38314"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:45.079224778Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1723,"client":"1","":"downstairs","session_id":"87ce64a4-b362-4c5a-9fd8-10964fa38314"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:45.079245048Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1723,"client":"0","":"downstairs","session_id":"87ce64a4-b362-4c5a-9fd8-10964fa38314"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:45.079256148Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1723,"client":"2","":"downstairs","session_id":"87ce64a4-b362-4c5a-9fd8-10964fa38314"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:45.079549469Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1723,"session_id":"87ce64a4-b362-4c5a-9fd8-10964fa38314"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:45.079594219Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1723,"session_id":"87ce64a4-b362-4c5a-9fd8-10964fa38314"} {"msg":"[0]R flush_numbers[0..12]: [240, 240, 242, 238, 234, 242, 242, 239, 242, 242, 240, 238]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:45.079639829Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1723,"":"downstairs","":"downstairs","session_id":"87ce64a4-b362-4c5a-9fd8-10964fa38314"} {"msg":"[0]R generation[0..12]: [143, 143, 144, 141, 138, 144, 144, 142, 144, 144, 143, 141]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:45.079651369Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1723,"":"downstairs","":"downstairs","session_id":"87ce64a4-b362-4c5a-9fd8-10964fa38314"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:45.079662779Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1723,"":"downstairs","":"downstairs","session_id":"87ce64a4-b362-4c5a-9fd8-10964fa38314"} {"msg":"[1]R flush_numbers[0..12]: [240, 240, 242, 238, 234, 242, 242, 239, 242, 242, 240, 238]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:45.079675639Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1723,"":"downstairs","":"downstairs","session_id":"87ce64a4-b362-4c5a-9fd8-10964fa38314"} {"msg":"[1]R generation[0..12]: [143, 143, 144, 141, 138, 144, 144, 142, 144, 144, 143, 141]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:45.079686509Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1723,"":"downstairs","":"downstairs","session_id":"87ce64a4-b362-4c5a-9fd8-10964fa38314"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:45.079697559Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1723,"":"downstairs","":"downstairs","session_id":"87ce64a4-b362-4c5a-9fd8-10964fa38314"} {"msg":"[2]R flush_numbers[0..12]: [240, 240, 242, 238, 234, 242, 242, 239, 242, 242, 240, 238]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:45.079708389Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1723,"":"downstairs","":"downstairs","session_id":"87ce64a4-b362-4c5a-9fd8-10964fa38314"} {"msg":"[2]R generation[0..12]: [143, 143, 144, 141, 138, 144, 144, 142, 144, 144, 143, 141]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:45.079719249Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1723,"":"downstairs","":"downstairs","session_id":"87ce64a4-b362-4c5a-9fd8-10964fa38314"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:45.079730259Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1723,"":"downstairs","":"downstairs","session_id":"87ce64a4-b362-4c5a-9fd8-10964fa38314"} {"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:45.079739859Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1723,"":"downstairs","":"downstairs","session_id":"87ce64a4-b362-4c5a-9fd8-10964fa38314"} {"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:45.079749739Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1723,"":"downstairs","":"downstairs","session_id":"87ce64a4-b362-4c5a-9fd8-10964fa38314"} {"msg":"Next flush: 244","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:45.079759459Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1723,"":"downstairs","":"downstairs","session_id":"87ce64a4-b362-4c5a-9fd8-10964fa38314"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:45.079769049Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1723,"":"downstairs","":"downstairs","session_id":"87ce64a4-b362-4c5a-9fd8-10964fa38314"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:45.079778859Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1723,"session_id":"87ce64a4-b362-4c5a-9fd8-10964fa38314"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:45.079787879Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1723,"client":"0","":"downstairs","session_id":"87ce64a4-b362-4c5a-9fd8-10964fa38314"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:45.079797749Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1723,"client":"1","":"downstairs","session_id":"87ce64a4-b362-4c5a-9fd8-10964fa38314"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:45.079807579Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1723,"client":"2","":"downstairs","session_id":"87ce64a4-b362-4c5a-9fd8-10964fa38314"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:45.079817529Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1723,"session_id":"87ce64a4-b362-4c5a-9fd8-10964fa38314"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:45.079828079Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1723,"session_id":"87ce64a4-b362-4c5a-9fd8-10964fa38314"} {"msg":"d78f172e-d6f6-462d-b161-b956d46bef8c is now active with session: 87ce64a4-b362-4c5a-9fd8-10964fa38314","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:45.079838149Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1723,"session_id":"87ce64a4-b362-4c5a-9fd8-10964fa38314"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:45.079846989Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1723,"session_id":"87ce64a4-b362-4c5a-9fd8-10964fa38314"} {"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:45.079856319Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1723} {"msg":"Activated sub_volume d78f172e-d6f6-462d-b161-b956d46bef8c","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:45.079864469Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1723} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:45.079872699Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1723,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:45.08014927Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1723,"task":"crutest"} 01/30 Write block 169 len 8192 data: 12 13 02/30 Write block 304 len 28672 data: 14 12 11 10 7 7 8 03/30 Read block 182 len 28672 04/30 Read block 79 len 20480 05/30 Read block 574 len 40960 06/30 Write block 53 len 8192 data: 12 11 07/30 Read block 450 len 16384 08/30 Write block 191 len 28672 data: 14 15 15 11 12 11 12 09/30 Write block 28 len 40960 data: 15 13 15 15 12 10 8 6 7 6 10/30 Write block 279 len 16384 data: 9 12 10 9 11/30 Flush 12/30 Write block 91 len 40960 data: 15 14 12 12 12 10 11 11 7 7 13/30 Write block 223 len 16384 data: 8 6 8 8 14/30 Read block 388 len 36864 15/30 Write block 69 len 12288 data: 10 12 9 16/30 Flush 17/30 Write block 218 len 40960 data: 11 10 8 7 7 9 7 9 9 9 18/30 Write block 144 len 24576 data: 11 9 8 9 9 11 19/30 Read block 3 len 28672 20/30 Write block 544 len 8192 data: 15 11 21/30 Read block 248 len 28672 22/30 Write block 332 len 12288 data: 12 9 9 23/30 Write block 128 len 8192 data: 8 7 24/30 Read block 119 len 24576 25/30 Write block 542 len 40960 data: 10 13 16 12 11 10 11 10 10 10 26/30 Write block 275 len 8192 data: 15 14 27/30 Write block 123 len 12288 data: 9 8 10 28/30 Read block 88 len 28672 29/30 Write block 568 len 8192 data: 12 14 30/30 Read block 476 len 16384 ---------------------------------------------------------------- Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Write 7 Done Done Done false 1002 Acked Read 7 Done Done Sent false 1003 Acked Read 5 Done Done Sent false 1004 Acked Read 10 Done Done Sent false 1005 Acked Write 2 Done Done Sent false 1006 Acked Read 4 Done Done Sent false 1007 Acked Write 7 Done Done Sent false 1008 Acked Write 10 Done Done Sent false 1009 Acked Write 4 Done Done Sent false 1010 Acked Flush 0 Done Done Sent false 1011 Acked Write 10 Done Done Sent false 1012 Acked Write 4 Done Done Sent false 1013 Acked Read 9 Done Done Sent false 1014 Acked Write 3 Done Done Sent false 1015 Acked Flush 0 Done Done Sent false 1016 Acked Write 10 Done Done Sent false 1017 Acked Write 6 Done Done Sent false 1018 Acked Read 7 Done Done Sent false 1019 Acked Write 2 Done Done Sent false 1020 Acked Read 7 Done Done Sent false 1021 Acked Write 3 Done Done Sent false 1022 Acked Write 2 Done Done Sent false 1023 Acked Read 6 Done Done Sent false 1024 Acked Write 10 Done Done Sent false 1025 Acked Write 2 Done Done Sent false 1026 Acked Write 3 Done Done Sent false 1027 Acked Read 7 Done Done Sent false 1028 Acked Write 2 Done Sent Sent false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 28 30 Done 30 28 2 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) Some(JobId(1015)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:24:45.088452842Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1723} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:45.088645063Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1723,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 143 143 143 240 240 240 F F F 1 020-039 146 146 143 244 244 240 F F F <--- 2 040-059 146 146 144 244 244 242 F F F <--- 3 060-079 146 146 141 245 245 238 F F F <--- 4 080-099 146 146 138 245 245 234 F F F <--- 5 100-119 146 146 144 245 245 242 F F F <--- 6 120-139 144 144 144 242 242 242 T T F <--- 7 140-159 142 142 142 239 239 239 T T F <--- 8 160-179 146 146 144 244 244 242 F F T <--- 9 180-199 146 146 144 244 244 242 F F F <--- 10 200-219 143 143 143 240 240 240 T T F <--- 11 220-239 146 146 141 245 245 238 T T F <--- 12 240-259 137 137 137 232 232 232 F F F 13 260-279 146 146 143 244 244 240 T T F <--- 14 280-299 146 146 143 244 244 240 F F F <--- 15 300-319 146 146 137 244 244 232 F F T <--- 16 320-339 142 142 142 239 239 239 T T F <--- 17 340-359 145 145 145 243 243 243 F F F 18 360-379 144 144 144 242 242 242 F F F 19 380-399 137 137 137 232 232 232 F F F 20 400-419 144 144 144 242 242 242 F F F 21 420-439 141 141 141 238 238 238 F F F 22 440-459 145 145 145 243 243 243 F F F 23 460-479 142 142 142 239 239 239 F F F 24 480-499 144 144 144 242 242 242 F F F 25 500-519 144 144 144 242 242 242 F F F 26 520-539 141 141 141 238 238 238 F F F 27 540-559 144 144 144 241 241 241 T T F <--- 28 560-579 145 145 145 243 243 243 T T F <--- 29 580-599 136 136 136 231 231 231 F F F Max gen: 146, Max flush: 245 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 73 Verifying data now No change after verify Loop: 73 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 143 143 143 240 240 240 F F F 1 020-039 146 146 146 244 244 244 F F F 2 040-059 146 146 146 244 244 244 F F F 3 060-079 146 146 146 245 245 245 F F F 4 080-099 146 146 146 245 245 245 F F F 5 100-119 146 146 146 245 245 245 F F F 6 120-139 147 147 147 246 246 246 F F F 7 140-159 147 147 147 246 246 246 F F F 8 160-179 146 146 146 244 244 244 F F F 9 180-199 146 146 146 244 244 244 F F F 10 200-219 147 147 147 246 246 246 F F F 11 220-239 147 147 147 246 246 246 F F F 12 240-259 137 137 137 232 232 232 F F F 13 260-279 147 147 147 246 246 246 F F F 14 280-299 146 146 146 244 244 244 F F F 15 300-319 146 146 146 244 244 244 F F F 16 320-339 147 147 147 246 246 246 F F F 17 340-359 145 145 145 243 243 243 F F F 18 360-379 144 144 144 242 242 242 F F F 19 380-399 137 137 137 232 232 232 F F F 20 400-419 144 144 144 242 242 242 F F F 21 420-439 141 141 141 238 238 238 F F F 22 440-459 145 145 145 243 243 243 F F F 23 460-479 142 142 142 239 239 239 F F F 24 480-499 144 144 144 242 242 242 F F F 25 500-519 144 144 144 242 242 242 F F F 26 520-539 141 141 141 238 238 238 F F F 27 540-559 147 147 147 246 246 246 F F F 28 560-579 147 147 147 246 246 246 F F F 29 580-599 136 136 136 231 231 231 F F F Max gen: 147, Max flush: 246 Begin loop 74 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:49.349579972Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1731,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:49.357618705Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1731,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:49.357899235Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1731} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:49.357933815Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1731} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:49.357943075Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1731} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:49.357963305Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1731} {"msg":"Crucible fab37896-7eb2-40db-ad19-60befafbaee0 has session id: 6c11cc1a-e0d2-4ce4-b1f9-f6ff18fa3b35","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:49.358215905Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1731,"session_id":"6c11cc1a-e0d2-4ce4-b1f9-f6ff18fa3b35"} {"msg":"Upstairs opts: Upstairs UUID: fab37896-7eb2-40db-ad19-60befafbaee0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:49.358275706Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1731,"session_id":"6c11cc1a-e0d2-4ce4-b1f9-f6ff18fa3b35"} {"msg":"Crucible stats registered with UUID: fab37896-7eb2-40db-ad19-60befafbaee0","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:49.358301556Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1731,"session_id":"6c11cc1a-e0d2-4ce4-b1f9-f6ff18fa3b35"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:24:49.358311706Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1731,"session_id":"6c11cc1a-e0d2-4ce4-b1f9-f6ff18fa3b35"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:49.358430016Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1731,"":"io task","client":"1","":"downstairs","session_id":"6c11cc1a-e0d2-4ce4-b1f9-f6ff18fa3b35"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:49.358460796Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1731,"":"io task","client":"0","":"downstairs","session_id":"6c11cc1a-e0d2-4ce4-b1f9-f6ff18fa3b35"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:49.358471626Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1731,"":"io task","client":"2","":"downstairs","session_id":"6c11cc1a-e0d2-4ce4-b1f9-f6ff18fa3b35"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:49.358603076Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1731,"task":"crutest"} {"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:49.358622306Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1731} {"msg":"fab37896-7eb2-40db-ad19-60befafbaee0 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:49.358651576Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1731,"session_id":"6c11cc1a-e0d2-4ce4-b1f9-f6ff18fa3b35"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:49.358661116Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1731,"client":"0","":"downstairs","session_id":"6c11cc1a-e0d2-4ce4-b1f9-f6ff18fa3b35"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:49.358670896Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1731,"client":"1","":"downstairs","session_id":"6c11cc1a-e0d2-4ce4-b1f9-f6ff18fa3b35"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:49.358680586Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1731,"client":"2","":"downstairs","session_id":"6c11cc1a-e0d2-4ce4-b1f9-f6ff18fa3b35"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:49.358690196Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1731,"":"io task","client":"2","":"downstairs","session_id":"6c11cc1a-e0d2-4ce4-b1f9-f6ff18fa3b35"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:49.358700326Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1731,"":"io task","client":"0","":"downstairs","session_id":"6c11cc1a-e0d2-4ce4-b1f9-f6ff18fa3b35"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:49.358710256Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1731,"":"io task","client":"1","":"downstairs","session_id":"6c11cc1a-e0d2-4ce4-b1f9-f6ff18fa3b35"} {"msg":"ds_connection connected from Ok(127.0.0.1:59935)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:49.358775066Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1731,"":"io task","client":"1","":"downstairs","session_id":"6c11cc1a-e0d2-4ce4-b1f9-f6ff18fa3b35"} {"msg":"ds_connection connected from Ok(127.0.0.1:42843)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:49.358800806Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1731,"":"io task","client":"2","":"downstairs","session_id":"6c11cc1a-e0d2-4ce4-b1f9-f6ff18fa3b35"} {"msg":"ds_connection connected from Ok(127.0.0.1:43321)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:49.358814286Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1731,"":"io task","client":"0","":"downstairs","session_id":"6c11cc1a-e0d2-4ce4-b1f9-f6ff18fa3b35"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:49.359410627Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1731,"client":"1","":"downstairs","session_id":"6c11cc1a-e0d2-4ce4-b1f9-f6ff18fa3b35"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:49.359452287Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1731,"client":"0","":"downstairs","session_id":"6c11cc1a-e0d2-4ce4-b1f9-f6ff18fa3b35"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:49.359536937Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1731,"client":"2","":"downstairs","session_id":"6c11cc1a-e0d2-4ce4-b1f9-f6ff18fa3b35"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:49.359710898Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1731,"session_id":"6c11cc1a-e0d2-4ce4-b1f9-f6ff18fa3b35"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:49.359732128Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1731,"session_id":"6c11cc1a-e0d2-4ce4-b1f9-f6ff18fa3b35"} {"msg":"[0]R flush_numbers[0..12]: [240, 244, 244, 245, 245, 245, 246, 246, 244, 244, 246, 246]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:49.359798768Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1731,"":"downstairs","":"downstairs","session_id":"6c11cc1a-e0d2-4ce4-b1f9-f6ff18fa3b35"} {"msg":"[0]R generation[0..12]: [143, 146, 146, 146, 146, 146, 147, 147, 146, 146, 147, 147]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:49.359810628Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1731,"":"downstairs","":"downstairs","session_id":"6c11cc1a-e0d2-4ce4-b1f9-f6ff18fa3b35"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:49.359821858Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1731,"":"downstairs","":"downstairs","session_id":"6c11cc1a-e0d2-4ce4-b1f9-f6ff18fa3b35"} {"msg":"[1]R flush_numbers[0..12]: [240, 244, 244, 245, 245, 245, 246, 246, 244, 244, 246, 246]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:49.359832708Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1731,"":"downstairs","":"downstairs","session_id":"6c11cc1a-e0d2-4ce4-b1f9-f6ff18fa3b35"} {"msg":"[1]R generation[0..12]: [143, 146, 146, 146, 146, 146, 147, 147, 146, 146, 147, 147]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:49.359843488Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1731,"":"downstairs","":"downstairs","session_id":"6c11cc1a-e0d2-4ce4-b1f9-f6ff18fa3b35"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:49.359854648Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1731,"":"downstairs","":"downstairs","session_id":"6c11cc1a-e0d2-4ce4-b1f9-f6ff18fa3b35"} {"msg":"[2]R flush_numbers[0..12]: [240, 244, 244, 245, 245, 245, 246, 246, 244, 244, 246, 246]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:49.359865488Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1731,"":"downstairs","":"downstairs","session_id":"6c11cc1a-e0d2-4ce4-b1f9-f6ff18fa3b35"} {"msg":"[2]R generation[0..12]: [143, 146, 146, 146, 146, 146, 147, 147, 146, 146, 147, 147]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:49.359878778Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1731,"":"downstairs","":"downstairs","session_id":"6c11cc1a-e0d2-4ce4-b1f9-f6ff18fa3b35"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:49.359889898Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1731,"":"downstairs","":"downstairs","session_id":"6c11cc1a-e0d2-4ce4-b1f9-f6ff18fa3b35"} {"msg":"Max found gen is 148","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:49.359899358Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1731,"":"downstairs","":"downstairs","session_id":"6c11cc1a-e0d2-4ce4-b1f9-f6ff18fa3b35"} {"msg":"Generation requested: 148 >= found:148","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:49.359909128Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1731,"":"downstairs","":"downstairs","session_id":"6c11cc1a-e0d2-4ce4-b1f9-f6ff18fa3b35"} {"msg":"Next flush: 247","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:49.359918588Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1731,"":"downstairs","":"downstairs","session_id":"6c11cc1a-e0d2-4ce4-b1f9-f6ff18fa3b35"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:49.359928148Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1731,"":"downstairs","":"downstairs","session_id":"6c11cc1a-e0d2-4ce4-b1f9-f6ff18fa3b35"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:49.359937958Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1731,"session_id":"6c11cc1a-e0d2-4ce4-b1f9-f6ff18fa3b35"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:49.359946878Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1731,"client":"0","":"downstairs","session_id":"6c11cc1a-e0d2-4ce4-b1f9-f6ff18fa3b35"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:49.359956768Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1731,"client":"1","":"downstairs","session_id":"6c11cc1a-e0d2-4ce4-b1f9-f6ff18fa3b35"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:49.359966648Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1731,"client":"2","":"downstairs","session_id":"6c11cc1a-e0d2-4ce4-b1f9-f6ff18fa3b35"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:49.359976598Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1731,"session_id":"6c11cc1a-e0d2-4ce4-b1f9-f6ff18fa3b35"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:49.359985808Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1731,"session_id":"6c11cc1a-e0d2-4ce4-b1f9-f6ff18fa3b35"} {"msg":"fab37896-7eb2-40db-ad19-60befafbaee0 is now active with session: 6c11cc1a-e0d2-4ce4-b1f9-f6ff18fa3b35","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:49.359996038Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1731,"session_id":"6c11cc1a-e0d2-4ce4-b1f9-f6ff18fa3b35"} {"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:49.360005458Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1731} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:49.360013278Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1731,"session_id":"6c11cc1a-e0d2-4ce4-b1f9-f6ff18fa3b35"} {"msg":"Activated sub_volume fab37896-7eb2-40db-ad19-60befafbaee0","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:49.360022458Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1731} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:49.360030688Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1731,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:49.360286999Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1731,"task":"crutest"} 01/30 Write block 3 len 20480 data: 9 9 9 11 11 02/30 Read block 51 len 16384 03/30 Read block 94 len 12288 04/30 Read block 111 len 12288 05/30 Write block 314 len 24576 data: 8 7 8 10 11 12 06/30 Read block 249 len 32768 07/30 Read block 516 len 36864 08/30 Read block 117 len 28672 09/30 Read block 454 len 12288 10/30 Write block 377 len 40960 data: 14 13 15 14 14 11 12 12 13 13 11/30 Read block 389 len 24576 12/30 Write block 502 len 32768 data: 14 12 9 11 12 8 8 8 13/30 Read block 317 len 40960 14/30 Read block 94 len 8192 15/30 Read block 510 len 16384 16/30 Read block 490 len 24576 17/30 Flush 18/30 Write block 208 len 36864 data: 11 12 12 12 11 9 8 9 10 19/30 Write block 370 len 20480 data: 10 12 14 11 12 20/30 Read block 150 len 12288 21/30 Write block 531 len 16384 data: 10 10 11 9 22/30 Read block 343 len 24576 23/30 Flush 24/30 Write block 184 len 28672 data: 10 14 14 16 16 13 12 25/30 Read block 236 len 32768 26/30 Read block 236 len 16384 27/30 Write block 111 len 28672 data: 11 11 12 12 12 12 9 28/30 Read block 364 len 40960 29/30 Read block 305 len 32768 30/30 Write block 213 len 28672 data: 10 9 10 11 11 12 11 ---------------------------------------------------------------- Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Read 4 Done Done Sent false 1002 Acked Read 3 Done Done Sent false 1003 Acked Read 3 Done Done Sent false 1004 Acked Write 6 Done Done Sent false 1005 Acked Read 8 Done Done Sent false 1006 Acked Read 9 Done Done Sent false 1007 Acked Read 7 Done Done Sent false 1008 Acked Read 3 Done Done Sent false 1009 Acked Write 10 Done Done Sent false 1010 Acked Read 6 Done Done Sent false 1011 Acked Write 8 Done Done Sent false 1012 Acked Read 10 Done Done Sent false 1013 Acked Read 2 Done Done Sent false 1014 Acked Read 4 Done Done Sent false 1015 Acked Read 6 Done Done Sent false 1016 Acked Flush 0 Done Done Sent false 1017 Acked Write 9 Done Done Sent false 1018 Acked Write 5 Done Done Sent false 1019 Acked Read 3 Done Done Sent false 1020 Acked Write 4 Done Done Sent false 1021 Acked Read 6 Done Done Sent false 1022 Acked Flush 0 Done Done Sent false 1023 Acked Write 7 Done Done Sent false 1024 Acked Read 8 Done Done Sent false 1025 Acked Read 4 Done Done Sent false 1026 Acked Write 7 Done Done Sent false 1027 Acked Read 10 Done Done Sent false 1028 Acked Read 8 Done Done Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 29 31 Done 29 29 1 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:24:49.369841773Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1731} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:49.370053453Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1731,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 143 247 247 240 F F T <--- 1 020-039 146 146 146 244 244 244 F F F 2 040-059 146 146 146 244 244 244 F F F 3 060-079 146 146 146 245 245 245 F F F 4 080-099 146 146 146 245 245 245 F F F 5 100-119 146 146 146 245 245 245 T T F <--- 6 120-139 147 147 147 246 246 246 F F F 7 140-159 147 147 147 246 246 246 F F F 8 160-179 146 146 146 244 244 244 F F F 9 180-199 146 146 146 244 244 244 T T F <--- 10 200-219 148 148 147 248 248 246 T T F <--- 11 220-239 147 147 147 246 246 246 F F F 12 240-259 137 137 137 232 232 232 F F F 13 260-279 147 147 147 246 246 246 F F F 14 280-299 146 146 146 244 244 244 F F F 15 300-319 148 148 146 247 247 244 F F F <--- 16 320-339 147 147 147 246 246 246 F F F 17 340-359 145 145 145 243 243 243 F F F 18 360-379 148 148 144 248 248 242 F F F <--- 19 380-399 148 148 137 247 247 232 F F F <--- 20 400-419 144 144 144 242 242 242 F F F 21 420-439 141 141 141 238 238 238 F F F 22 440-459 145 145 145 243 243 243 F F F 23 460-479 142 142 142 239 239 239 F F F 24 480-499 144 144 144 242 242 242 F F F 25 500-519 148 148 144 247 247 242 F F F <--- 26 520-539 148 148 141 248 248 238 F F F <--- 27 540-559 147 147 147 246 246 246 F F F 28 560-579 147 147 147 246 246 246 F F F 29 580-599 136 136 136 231 231 231 F F F Max gen: 148, Max flush: 248 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 74 Verifying data now No change after verify Loop: 74 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 247 247 247 F F F 1 020-039 146 146 146 244 244 244 F F F 2 040-059 146 146 146 244 244 244 F F F 3 060-079 146 146 146 245 245 245 F F F 4 080-099 146 146 146 245 245 245 F F F 5 100-119 149 149 149 249 249 249 F F F 6 120-139 147 147 147 246 246 246 F F F 7 140-159 147 147 147 246 246 246 F F F 8 160-179 146 146 146 244 244 244 F F F 9 180-199 149 149 149 249 249 249 F F F 10 200-219 149 149 149 249 249 249 F F F 11 220-239 147 147 147 246 246 246 F F F 12 240-259 137 137 137 232 232 232 F F F 13 260-279 147 147 147 246 246 246 F F F 14 280-299 146 146 146 244 244 244 F F F 15 300-319 148 148 148 247 247 247 F F F 16 320-339 147 147 147 246 246 246 F F F 17 340-359 145 145 145 243 243 243 F F F 18 360-379 148 148 148 248 248 248 F F F 19 380-399 148 148 148 247 247 247 F F F 20 400-419 144 144 144 242 242 242 F F F 21 420-439 141 141 141 238 238 238 F F F 22 440-459 145 145 145 243 243 243 F F F 23 460-479 142 142 142 239 239 239 F F F 24 480-499 144 144 144 242 242 242 F F F 25 500-519 148 148 148 247 247 247 F F F 26 520-539 148 148 148 248 248 248 F F F 27 540-559 147 147 147 246 246 246 F F F 28 560-579 147 147 147 246 246 246 F F F 29 580-599 136 136 136 231 231 231 F F F Max gen: 149, Max flush: 249 Begin loop 75 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:53.609676911Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1739,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:53.617941393Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1739,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:53.618236474Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1739} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:53.618265304Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1739} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:53.618274084Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1739} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:53.618285924Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1739} {"msg":"Crucible 5783c03e-e0ed-4222-8e10-5d300d9b0d3b has session id: 6565120f-ede6-4e8e-9a76-18ba91e83cd6","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:53.618668385Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1739,"session_id":"6565120f-ede6-4e8e-9a76-18ba91e83cd6"} {"msg":"Upstairs opts: Upstairs UUID: 5783c03e-e0ed-4222-8e10-5d300d9b0d3b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:53.618691515Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1739,"session_id":"6565120f-ede6-4e8e-9a76-18ba91e83cd6"} {"msg":"Crucible stats registered with UUID: 5783c03e-e0ed-4222-8e10-5d300d9b0d3b","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:53.618701265Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1739,"session_id":"6565120f-ede6-4e8e-9a76-18ba91e83cd6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:24:53.618710475Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1739,"session_id":"6565120f-ede6-4e8e-9a76-18ba91e83cd6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:53.618873655Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1739,"":"io task","client":"2","":"downstairs","session_id":"6565120f-ede6-4e8e-9a76-18ba91e83cd6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:53.618903915Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1739,"":"io task","client":"0","":"downstairs","session_id":"6565120f-ede6-4e8e-9a76-18ba91e83cd6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:53.618914875Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1739,"":"io task","client":"1","":"downstairs","session_id":"6565120f-ede6-4e8e-9a76-18ba91e83cd6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:53.619069555Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1739,"task":"crutest"} {"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:53.619088855Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1739} {"msg":"5783c03e-e0ed-4222-8e10-5d300d9b0d3b active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:53.619112435Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1739,"session_id":"6565120f-ede6-4e8e-9a76-18ba91e83cd6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:53.619121785Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1739,"client":"0","":"downstairs","session_id":"6565120f-ede6-4e8e-9a76-18ba91e83cd6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:53.619131585Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1739,"client":"1","":"downstairs","session_id":"6565120f-ede6-4e8e-9a76-18ba91e83cd6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:53.619141255Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1739,"client":"2","":"downstairs","session_id":"6565120f-ede6-4e8e-9a76-18ba91e83cd6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:53.619150935Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1739,"":"io task","client":"2","":"downstairs","session_id":"6565120f-ede6-4e8e-9a76-18ba91e83cd6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:53.619160985Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1739,"":"io task","client":"0","":"downstairs","session_id":"6565120f-ede6-4e8e-9a76-18ba91e83cd6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:53.619171015Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1739,"":"io task","client":"1","":"downstairs","session_id":"6565120f-ede6-4e8e-9a76-18ba91e83cd6"} {"msg":"ds_connection connected from Ok(127.0.0.1:58494)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:53.619224435Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1739,"":"io task","client":"2","":"downstairs","session_id":"6565120f-ede6-4e8e-9a76-18ba91e83cd6"} {"msg":"ds_connection connected from Ok(127.0.0.1:46902)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:53.619246185Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1739,"":"io task","client":"1","":"downstairs","session_id":"6565120f-ede6-4e8e-9a76-18ba91e83cd6"} {"msg":"ds_connection connected from Ok(127.0.0.1:55645)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:53.619347216Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1739,"":"io task","client":"0","":"downstairs","session_id":"6565120f-ede6-4e8e-9a76-18ba91e83cd6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:53.619908546Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1739,"client":"0","":"downstairs","session_id":"6565120f-ede6-4e8e-9a76-18ba91e83cd6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:53.619932206Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1739,"client":"1","":"downstairs","session_id":"6565120f-ede6-4e8e-9a76-18ba91e83cd6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:53.619997957Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1739,"client":"2","":"downstairs","session_id":"6565120f-ede6-4e8e-9a76-18ba91e83cd6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:53.620222797Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1739,"session_id":"6565120f-ede6-4e8e-9a76-18ba91e83cd6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:53.620244497Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1739,"session_id":"6565120f-ede6-4e8e-9a76-18ba91e83cd6"} {"msg":"[0]R flush_numbers[0..12]: [247, 244, 244, 245, 245, 249, 246, 246, 244, 249, 249, 246]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:53.620335807Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1739,"":"downstairs","":"downstairs","session_id":"6565120f-ede6-4e8e-9a76-18ba91e83cd6"} {"msg":"[0]R generation[0..12]: [148, 146, 146, 146, 146, 149, 147, 147, 146, 149, 149, 147]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:53.620347567Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1739,"":"downstairs","":"downstairs","session_id":"6565120f-ede6-4e8e-9a76-18ba91e83cd6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:53.620358967Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1739,"":"downstairs","":"downstairs","session_id":"6565120f-ede6-4e8e-9a76-18ba91e83cd6"} {"msg":"[1]R flush_numbers[0..12]: [247, 244, 244, 245, 245, 249, 246, 246, 244, 249, 249, 246]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:53.620369857Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1739,"":"downstairs","":"downstairs","session_id":"6565120f-ede6-4e8e-9a76-18ba91e83cd6"} {"msg":"[1]R generation[0..12]: [148, 146, 146, 146, 146, 149, 147, 147, 146, 149, 149, 147]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:53.620380557Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1739,"":"downstairs","":"downstairs","session_id":"6565120f-ede6-4e8e-9a76-18ba91e83cd6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:53.620391757Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1739,"":"downstairs","":"downstairs","session_id":"6565120f-ede6-4e8e-9a76-18ba91e83cd6"} {"msg":"[2]R flush_numbers[0..12]: [247, 244, 244, 245, 245, 249, 246, 246, 244, 249, 249, 246]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:53.620402377Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1739,"":"downstairs","":"downstairs","session_id":"6565120f-ede6-4e8e-9a76-18ba91e83cd6"} {"msg":"[2]R generation[0..12]: [148, 146, 146, 146, 146, 149, 147, 147, 146, 149, 149, 147]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:53.620413037Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1739,"":"downstairs","":"downstairs","session_id":"6565120f-ede6-4e8e-9a76-18ba91e83cd6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:53.620424037Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1739,"":"downstairs","":"downstairs","session_id":"6565120f-ede6-4e8e-9a76-18ba91e83cd6"} {"msg":"Max found gen is 150","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:53.620433497Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1739,"":"downstairs","":"downstairs","session_id":"6565120f-ede6-4e8e-9a76-18ba91e83cd6"} {"msg":"Generation requested: 150 >= found:150","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:53.620443357Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1739,"":"downstairs","":"downstairs","session_id":"6565120f-ede6-4e8e-9a76-18ba91e83cd6"} {"msg":"Next flush: 250","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:53.620452907Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1739,"":"downstairs","":"downstairs","session_id":"6565120f-ede6-4e8e-9a76-18ba91e83cd6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:53.620462377Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1739,"":"downstairs","":"downstairs","session_id":"6565120f-ede6-4e8e-9a76-18ba91e83cd6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:53.620474527Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1739,"session_id":"6565120f-ede6-4e8e-9a76-18ba91e83cd6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:53.620483537Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1739,"client":"0","":"downstairs","session_id":"6565120f-ede6-4e8e-9a76-18ba91e83cd6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:53.620493217Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1739,"client":"1","":"downstairs","session_id":"6565120f-ede6-4e8e-9a76-18ba91e83cd6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:53.620502867Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1739,"client":"2","":"downstairs","session_id":"6565120f-ede6-4e8e-9a76-18ba91e83cd6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:53.620512617Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1739,"session_id":"6565120f-ede6-4e8e-9a76-18ba91e83cd6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:53.620521967Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1739,"session_id":"6565120f-ede6-4e8e-9a76-18ba91e83cd6"} {"msg":"5783c03e-e0ed-4222-8e10-5d300d9b0d3b is now active with session: 6565120f-ede6-4e8e-9a76-18ba91e83cd6","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:53.620532027Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1739,"session_id":"6565120f-ede6-4e8e-9a76-18ba91e83cd6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:53.620540947Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1739,"session_id":"6565120f-ede6-4e8e-9a76-18ba91e83cd6"} {"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:53.620550237Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1739} {"msg":"Activated sub_volume 5783c03e-e0ed-4222-8e10-5d300d9b0d3b","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:53.620558367Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1739} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:53.620566487Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1739,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:53.620827198Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1739,"task":"crutest"} 01/30 Write block 69 len 28672 data: 11 13 10 9 9 8 7 02/30 Read block 279 len 4096 03/30 Read block 86 len 28672 04/30 Read block 571 len 24576 05/30 Write block 337 len 12288 data: 13 12 14 06/30 Write block 460 len 36864 data: 12 12 11 14 12 10 10 12 13 07/30 Write block 556 len 32768 data: 9 12 8 9 9 10 8 9 08/30 Read block 70 len 12288 09/30 Write block 459 len 28672 data: 12 13 13 12 15 13 11 10/30 Write block 189 len 8192 data: 14 13 11/30 Write block 236 len 28672 data: 11 14 13 15 15 16 14 12/30 Read block 63 len 20480 13/30 Read block 591 len 32768 14/30 Write block 583 len 20480 data: 10 11 9 10 11 15/30 Flush 16/30 Write block 587 len 8192 data: 12 11 17/30 Read block 527 len 4096 18/30 Read block 378 len 40960 19/30 Read block 388 len 16384 20/30 Read block 169 len 16384 21/30 Read block 549 len 12288 22/30 Read block 10 len 12288 23/30 Read block 266 len 36864 24/30 Read block 263 len 36864 25/30 Read block 416 len 20480 26/30 Read block 571 len 8192 27/30 Write block 267 len 20480 data: 15 17 17 16 18 28/30 Read block 392 len 28672 29/30 Write block 444 len 4096 data: 8 30/30 Read block 92 len 12288 ---------------------------------------------------------------- Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Read 1 Done Done Done false 1002 Acked Read 7 Done Done Done false 1003 Acked Read 6 Done Done Done false 1004 Acked Write 3 Done Done Done false 1005 Acked Write 9 Done Done Sent false 1006 Acked Write 8 Done Done Sent false 1007 Acked Read 3 Done Done Sent false 1008 Acked Write 7 Done Done Sent false 1009 Acked Write 2 Done Done Sent false 1010 Acked Write 7 Done Done Sent false 1011 Acked Read 5 Done Done Sent false 1012 Acked Read 8 Done Done Sent false 1013 Acked Write 5 Done Done Sent false 1014 Acked Flush 0 Done Done Sent false 1015 Acked Write 2 Done Done Sent false 1016 Acked Read 1 Done Done Sent false 1017 Acked Read 10 Done Done Sent false 1018 Acked Read 4 Done Done Sent false 1019 Acked Read 4 Done Done Sent false 1020 Acked Read 3 Done Done Sent false 1021 Acked Read 3 Done Done Sent false 1022 Acked Read 9 Done Done Sent false 1023 Acked Read 9 Done Done Sent false 1024 Acked Read 5 Done Done Sent false 1025 Acked Read 2 Done Done Sent false 1026 Acked Write 5 Done Done Sent false 1027 Acked Read 7 Done Done Sent false 1028 Acked Write 1 Done Done Sent false 1029 Acked Read 3 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 25 26 Done 29 30 5 64 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) Some(JobId(1014)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:24:53.629423001Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1739} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:53.629611021Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1739,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 247 247 247 F F F 1 020-039 146 146 146 244 244 244 F F F 2 040-059 146 146 146 244 244 244 F F F 3 060-079 150 150 146 250 250 245 F F T <--- 4 080-099 146 146 146 245 245 245 F F F 5 100-119 149 149 149 249 249 249 F F F 6 120-139 147 147 147 246 246 246 F F F 7 140-159 147 147 147 246 246 246 F F F 8 160-179 146 146 146 244 244 244 F F F 9 180-199 150 150 149 250 250 249 F F F <--- 10 200-219 149 149 149 249 249 249 F F F 11 220-239 150 150 147 250 250 246 F F F <--- 12 240-259 150 150 137 250 250 232 F F F <--- 13 260-279 147 147 147 246 246 246 T T F <--- 14 280-299 146 146 146 244 244 244 F F F 15 300-319 148 148 148 247 247 247 F F F 16 320-339 150 150 147 250 250 246 F F T <--- 17 340-359 145 145 145 243 243 243 F F F 18 360-379 148 148 148 248 248 248 F F F 19 380-399 148 148 148 247 247 247 F F F 20 400-419 144 144 144 242 242 242 F F F 21 420-439 141 141 141 238 238 238 F F F 22 440-459 150 150 145 250 250 243 T T F <--- 23 460-479 150 150 142 250 250 239 F F F <--- 24 480-499 144 144 144 242 242 242 F F F 25 500-519 148 148 148 247 247 247 F F F 26 520-539 148 148 148 248 248 248 F F F 27 540-559 150 150 147 250 250 246 F F F <--- 28 560-579 150 150 147 250 250 246 F F F <--- 29 580-599 150 150 136 250 250 231 T T F <--- Max gen: 150, Max flush: 250 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 75 Verifying data now No change after verify Loop: 75 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 247 247 247 F F F 1 020-039 146 146 146 244 244 244 F F F 2 040-059 146 146 146 244 244 244 F F F 3 060-079 150 150 150 250 250 250 F F F 4 080-099 146 146 146 245 245 245 F F F 5 100-119 149 149 149 249 249 249 F F F 6 120-139 147 147 147 246 246 246 F F F 7 140-159 147 147 147 246 246 246 F F F 8 160-179 146 146 146 244 244 244 F F F 9 180-199 150 150 150 250 250 250 F F F 10 200-219 149 149 149 249 249 249 F F F 11 220-239 150 150 150 250 250 250 F F F 12 240-259 150 150 150 250 250 250 F F F 13 260-279 151 151 151 251 251 251 F F F 14 280-299 146 146 146 244 244 244 F F F 15 300-319 148 148 148 247 247 247 F F F 16 320-339 150 150 150 250 250 250 F F F 17 340-359 145 145 145 243 243 243 F F F 18 360-379 148 148 148 248 248 248 F F F 19 380-399 148 148 148 247 247 247 F F F 20 400-419 144 144 144 242 242 242 F F F 21 420-439 141 141 141 238 238 238 F F F 22 440-459 151 151 151 251 251 251 F F F 23 460-479 150 150 150 250 250 250 F F F 24 480-499 144 144 144 242 242 242 F F F 25 500-519 148 148 148 247 247 247 F F F 26 520-539 148 148 148 248 248 248 F F F 27 540-559 150 150 150 250 250 250 F F F 28 560-579 150 150 150 250 250 250 F F F 29 580-599 151 151 151 251 251 251 F F F Max gen: 151, Max flush: 251 Begin loop 76 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:57.874764957Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1747,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:57.882844429Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1747,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:57.883141559Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1747} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:57.883166039Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1747} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:57.883177049Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1747} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:57.883189219Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1747} {"msg":"Crucible 0e0197cc-9515-4032-9b7a-c2d3c214381d has session id: 8874bd6e-a26e-4bb3-8437-f8aa64b56533","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:57.88355931Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1747,"session_id":"8874bd6e-a26e-4bb3-8437-f8aa64b56533"} {"msg":"Upstairs opts: Upstairs UUID: 0e0197cc-9515-4032-9b7a-c2d3c214381d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:57.88360835Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1747,"session_id":"8874bd6e-a26e-4bb3-8437-f8aa64b56533"} {"msg":"Crucible stats registered with UUID: 0e0197cc-9515-4032-9b7a-c2d3c214381d","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:57.8836189Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1747,"session_id":"8874bd6e-a26e-4bb3-8437-f8aa64b56533"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:24:57.8836283Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1747,"session_id":"8874bd6e-a26e-4bb3-8437-f8aa64b56533"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:57.88377639Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1747,"":"io task","client":"2","":"downstairs","session_id":"8874bd6e-a26e-4bb3-8437-f8aa64b56533"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:57.88380903Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1747,"":"io task","client":"0","":"downstairs","session_id":"8874bd6e-a26e-4bb3-8437-f8aa64b56533"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:57.88382013Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1747,"":"io task","client":"1","":"downstairs","session_id":"8874bd6e-a26e-4bb3-8437-f8aa64b56533"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:57.883959101Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1747,"task":"crutest"} {"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:57.883978771Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1747} {"msg":"0e0197cc-9515-4032-9b7a-c2d3c214381d active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:57.884004041Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1747,"session_id":"8874bd6e-a26e-4bb3-8437-f8aa64b56533"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:57.884013221Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1747,"client":"0","":"downstairs","session_id":"8874bd6e-a26e-4bb3-8437-f8aa64b56533"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:57.884022951Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1747,"client":"1","":"downstairs","session_id":"8874bd6e-a26e-4bb3-8437-f8aa64b56533"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:57.884032631Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1747,"client":"2","":"downstairs","session_id":"8874bd6e-a26e-4bb3-8437-f8aa64b56533"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:57.884044141Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1747,"":"io task","client":"2","":"downstairs","session_id":"8874bd6e-a26e-4bb3-8437-f8aa64b56533"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:57.884091441Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1747,"":"io task","client":"0","":"downstairs","session_id":"8874bd6e-a26e-4bb3-8437-f8aa64b56533"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:57.884115561Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1747,"":"io task","client":"1","":"downstairs","session_id":"8874bd6e-a26e-4bb3-8437-f8aa64b56533"} {"msg":"ds_connection connected from Ok(127.0.0.1:33276)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:57.884126901Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1747,"":"io task","client":"0","":"downstairs","session_id":"8874bd6e-a26e-4bb3-8437-f8aa64b56533"} {"msg":"ds_connection connected from Ok(127.0.0.1:55995)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:57.884137651Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1747,"":"io task","client":"2","":"downstairs","session_id":"8874bd6e-a26e-4bb3-8437-f8aa64b56533"} {"msg":"ds_connection connected from Ok(127.0.0.1:38578)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:57.884148111Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1747,"":"io task","client":"1","":"downstairs","session_id":"8874bd6e-a26e-4bb3-8437-f8aa64b56533"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:57.884776442Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1747,"client":"2","":"downstairs","session_id":"8874bd6e-a26e-4bb3-8437-f8aa64b56533"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:57.884798942Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1747,"client":"0","":"downstairs","session_id":"8874bd6e-a26e-4bb3-8437-f8aa64b56533"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:57.884879362Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1747,"client":"1","":"downstairs","session_id":"8874bd6e-a26e-4bb3-8437-f8aa64b56533"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:57.885081162Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1747,"session_id":"8874bd6e-a26e-4bb3-8437-f8aa64b56533"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:57.885100782Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1747,"session_id":"8874bd6e-a26e-4bb3-8437-f8aa64b56533"} {"msg":"[0]R flush_numbers[0..12]: [247, 244, 244, 250, 245, 249, 246, 246, 244, 250, 249, 250]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:57.885150482Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1747,"":"downstairs","":"downstairs","session_id":"8874bd6e-a26e-4bb3-8437-f8aa64b56533"} {"msg":"[0]R generation[0..12]: [148, 146, 146, 150, 146, 149, 147, 147, 146, 150, 149, 150]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:57.885161842Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1747,"":"downstairs","":"downstairs","session_id":"8874bd6e-a26e-4bb3-8437-f8aa64b56533"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:57.885172822Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1747,"":"downstairs","":"downstairs","session_id":"8874bd6e-a26e-4bb3-8437-f8aa64b56533"} {"msg":"[1]R flush_numbers[0..12]: [247, 244, 244, 250, 245, 249, 246, 246, 244, 250, 249, 250]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:57.885183732Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1747,"":"downstairs","":"downstairs","session_id":"8874bd6e-a26e-4bb3-8437-f8aa64b56533"} {"msg":"[1]R generation[0..12]: [148, 146, 146, 150, 146, 149, 147, 147, 146, 150, 149, 150]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:57.885197072Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1747,"":"downstairs","":"downstairs","session_id":"8874bd6e-a26e-4bb3-8437-f8aa64b56533"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:57.885208192Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1747,"":"downstairs","":"downstairs","session_id":"8874bd6e-a26e-4bb3-8437-f8aa64b56533"} {"msg":"[2]R flush_numbers[0..12]: [247, 244, 244, 250, 245, 249, 246, 246, 244, 250, 249, 250]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:57.885218922Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1747,"":"downstairs","":"downstairs","session_id":"8874bd6e-a26e-4bb3-8437-f8aa64b56533"} {"msg":"[2]R generation[0..12]: [148, 146, 146, 150, 146, 149, 147, 147, 146, 150, 149, 150]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:57.885229642Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1747,"":"downstairs","":"downstairs","session_id":"8874bd6e-a26e-4bb3-8437-f8aa64b56533"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:57.885240682Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1747,"":"downstairs","":"downstairs","session_id":"8874bd6e-a26e-4bb3-8437-f8aa64b56533"} {"msg":"Max found gen is 152","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:57.885250213Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1747,"":"downstairs","":"downstairs","session_id":"8874bd6e-a26e-4bb3-8437-f8aa64b56533"} {"msg":"Generation requested: 152 >= found:152","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:57.885259943Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1747,"":"downstairs","":"downstairs","session_id":"8874bd6e-a26e-4bb3-8437-f8aa64b56533"} {"msg":"Next flush: 252","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:57.885269353Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1747,"":"downstairs","":"downstairs","session_id":"8874bd6e-a26e-4bb3-8437-f8aa64b56533"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:57.885278763Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1747,"":"downstairs","":"downstairs","session_id":"8874bd6e-a26e-4bb3-8437-f8aa64b56533"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:57.885288523Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1747,"session_id":"8874bd6e-a26e-4bb3-8437-f8aa64b56533"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:57.885297333Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1747,"client":"0","":"downstairs","session_id":"8874bd6e-a26e-4bb3-8437-f8aa64b56533"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:57.885306963Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1747,"client":"1","":"downstairs","session_id":"8874bd6e-a26e-4bb3-8437-f8aa64b56533"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:57.885316593Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1747,"client":"2","":"downstairs","session_id":"8874bd6e-a26e-4bb3-8437-f8aa64b56533"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:57.885326383Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1747,"session_id":"8874bd6e-a26e-4bb3-8437-f8aa64b56533"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:57.885335553Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1747,"session_id":"8874bd6e-a26e-4bb3-8437-f8aa64b56533"} {"msg":"0e0197cc-9515-4032-9b7a-c2d3c214381d is now active with session: 8874bd6e-a26e-4bb3-8437-f8aa64b56533","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:57.885347083Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1747,"session_id":"8874bd6e-a26e-4bb3-8437-f8aa64b56533"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:57.885355933Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1747,"session_id":"8874bd6e-a26e-4bb3-8437-f8aa64b56533"} {"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:57.885365083Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1747} {"msg":"Activated sub_volume 0e0197cc-9515-4032-9b7a-c2d3c214381d","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:57.885373213Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1747} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:57.885381343Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1747,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:57.885705333Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1747,"task":"crutest"} 01/30 Write block 260 len 8192 data: 11 12 02/30 Write block 113 len 12288 data: 13 13 13 03/30 Read block 577 len 24576 04/30 Read block 65 len 40960 05/30 Read block 577 len 12288 06/30 Read block 11 len 28672 07/30 Flush 08/30 Write block 573 len 32768 data: 13 13 14 13 13 14 11 10 09/30 Read block 106 len 32768 10/30 Read block 301 len 4096 11/30 Read block 587 len 4096 12/30 Write block 297 len 12288 data: 14 16 15 13/30 Read block 267 len 20480 14/30 Flush 15/30 Write block 426 len 24576 data: 7 7 8 8 6 6 16/30 Write block 584 len 40960 data: 12 10 11 13 12 9 6 5 6 5 17/30 Read block 80 len 40960 18/30 Write block 418 len 4096 data: 10 19/30 Write block 275 len 24576 data: 16 15 15 11 10 13 20/30 Read block 416 len 24576 21/30 Read block 416 len 28672 22/30 Read block 178 len 28672 23/30 Write block 562 len 20480 data: 9 10 8 11 12 24/30 Read block 404 len 8192 25/30 Write block 52 len 40960 data: 14 13 12 13 9 10 12 12 12 15 26/30 Write block 463 len 16384 data: 16 14 12 11 27/30 Read block 105 len 4096 28/30 Write block 574 len 36864 data: 14 15 14 14 15 12 11 9 9 29/30 Read block 301 len 8192 30/30 Read block 517 len 16384 ---------------------------------------------------------------- Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Write 3 Done Done Done false 1002 Acked Read 6 Done Sent Done false 1003 Acked Read 10 Done Sent Done false 1004 Acked Read 3 Done Sent Done false 1005 Acked Read 7 Done Sent Done false 1006 Acked Flush 0 Done Sent Done false 1007 Acked Write 8 Done Sent Done false 1008 Acked Read 8 Done Sent Done false 1009 Acked Read 1 Done Sent Done false 1010 Acked Read 1 Done Sent Done false 1011 Acked Write 3 Done Sent Done false 1012 Acked Read 5 Done Sent Done false 1013 Acked Flush 0 Done Sent Done false 1014 Acked Write 6 Done Sent Done false 1015 Acked Write 10 Done Sent Done false 1016 Acked Read 10 Done Sent Done false 1017 Acked Write 1 Done Sent Done false 1018 Acked Write 6 Done Sent Done false 1019 Acked Read 6 Done Sent Done false 1020 Acked Read 7 Done Sent Done false 1021 Acked Read 7 Done Sent Done false 1022 Acked Write 5 Done Sent Done false 1023 Acked Read 2 Done Sent Done false 1024 Acked Write 10 Done Sent Done false 1025 Acked Write 4 Done Sent Done false 1026 Acked Read 1 Done Sent Done false 1027 Acked Write 9 Done Sent Done false 1028 Acked Read 2 Done Sent Done false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 28 1 29 Done 30 2 29 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1013)) None Some(JobId(1013)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:24:57.894224906Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1747} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:24:57.894407856Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1747,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 247 247 247 F F F 1 020-039 146 146 146 244 244 244 F F F 2 040-059 146 146 146 244 244 244 T F T <--- 3 060-079 150 150 150 250 250 250 T F T <--- 4 080-099 146 146 146 245 245 245 F F F 5 100-119 152 149 152 252 249 252 F T F <--- 6 120-139 147 147 147 246 246 246 F F F 7 140-159 147 147 147 246 246 246 F F F 8 160-179 146 146 146 244 244 244 F F F 9 180-199 150 150 150 250 250 250 F F F 10 200-219 149 149 149 249 249 249 F F F 11 220-239 150 150 150 250 250 250 F F F 12 240-259 150 150 150 250 250 250 F F F 13 260-279 152 151 152 252 251 252 T T T <--- 14 280-299 152 146 152 253 244 253 T F T <--- 15 300-319 148 148 148 247 247 247 F F F 16 320-339 150 150 150 250 250 250 F F F 17 340-359 145 145 145 243 243 243 F F F 18 360-379 148 148 148 248 248 248 F F F 19 380-399 148 148 148 247 247 247 F F F 20 400-419 144 144 144 242 242 242 T F T <--- 21 420-439 141 141 141 238 238 238 T F T <--- 22 440-459 151 151 151 251 251 251 F F F 23 460-479 150 150 150 250 250 250 T F T <--- 24 480-499 144 144 144 242 242 242 F F F 25 500-519 148 148 148 247 247 247 F F F 26 520-539 148 148 148 248 248 248 F F F 27 540-559 150 150 150 250 250 250 F F F 28 560-579 152 150 152 253 250 253 T F T <--- 29 580-599 152 151 152 253 251 253 T F T <--- Max gen: 152, Max flush: 253 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 76 Verifying data now No change after verify Loop: 76 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 247 247 247 F F F 1 020-039 146 146 146 244 244 244 F F F 2 040-059 153 153 153 254 254 254 F F F 3 060-079 153 153 153 254 254 254 F F F 4 080-099 146 146 146 245 245 245 F F F 5 100-119 152 152 152 252 252 252 F F F 6 120-139 147 147 147 246 246 246 F F F 7 140-159 147 147 147 246 246 246 F F F 8 160-179 146 146 146 244 244 244 F F F 9 180-199 150 150 150 250 250 250 F F F 10 200-219 149 149 149 249 249 249 F F F 11 220-239 150 150 150 250 250 250 F F F 12 240-259 150 150 150 250 250 250 F F F 13 260-279 153 153 153 254 254 254 F F F 14 280-299 153 153 153 254 254 254 F F F 15 300-319 148 148 148 247 247 247 F F F 16 320-339 150 150 150 250 250 250 F F F 17 340-359 145 145 145 243 243 243 F F F 18 360-379 148 148 148 248 248 248 F F F 19 380-399 148 148 148 247 247 247 F F F 20 400-419 153 153 153 254 254 254 F F F 21 420-439 153 153 153 254 254 254 F F F 22 440-459 151 151 151 251 251 251 F F F 23 460-479 153 153 153 254 254 254 F F F 24 480-499 144 144 144 242 242 242 F F F 25 500-519 148 148 148 247 247 247 F F F 26 520-539 148 148 148 248 248 248 F F F 27 540-559 150 150 150 250 250 250 F F F 28 560-579 153 153 153 254 254 254 F F F 29 580-599 153 153 153 254 254 254 F F F Max gen: 153, Max flush: 254 Begin loop 77 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:02.141972446Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1755,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:25:02.149178247Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1755,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:02.149247087Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1755,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:02.156133747Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1755,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:02.156403988Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1755} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:02.156429548Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1755} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:02.156438178Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1755} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:02.156449888Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1755} {"msg":"Crucible d8852b41-395f-4f88-89e2-56eda2b40891 has session id: 8ce6c17b-3f87-4531-a1f6-916f07a88fdf","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:02.156823878Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1755,"session_id":"8ce6c17b-3f87-4531-a1f6-916f07a88fdf"} {"msg":"Upstairs opts: Upstairs UUID: d8852b41-395f-4f88-89e2-56eda2b40891, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:02.156846568Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1755,"session_id":"8ce6c17b-3f87-4531-a1f6-916f07a88fdf"} {"msg":"Crucible stats registered with UUID: d8852b41-395f-4f88-89e2-56eda2b40891","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:02.156856618Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1755,"session_id":"8ce6c17b-3f87-4531-a1f6-916f07a88fdf"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:25:02.156865699Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1755,"session_id":"8ce6c17b-3f87-4531-a1f6-916f07a88fdf"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:02.157018249Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1755,"":"io task","client":"0","":"downstairs","session_id":"8ce6c17b-3f87-4531-a1f6-916f07a88fdf"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:02.157036629Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1755,"":"io task","client":"2","":"downstairs","session_id":"8ce6c17b-3f87-4531-a1f6-916f07a88fdf"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:02.157046849Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1755,"":"io task","client":"1","":"downstairs","session_id":"8ce6c17b-3f87-4531-a1f6-916f07a88fdf"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:02.157221829Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1755,"task":"crutest"} {"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:02.157239119Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1755} {"msg":"d8852b41-395f-4f88-89e2-56eda2b40891 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:02.157257089Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1755,"session_id":"8ce6c17b-3f87-4531-a1f6-916f07a88fdf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:02.157265989Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1755,"client":"0","":"downstairs","session_id":"8ce6c17b-3f87-4531-a1f6-916f07a88fdf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:02.157275709Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1755,"client":"1","":"downstairs","session_id":"8ce6c17b-3f87-4531-a1f6-916f07a88fdf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:02.157285339Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1755,"client":"2","":"downstairs","session_id":"8ce6c17b-3f87-4531-a1f6-916f07a88fdf"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:02.157294999Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1755,"":"io task","client":"2","":"downstairs","session_id":"8ce6c17b-3f87-4531-a1f6-916f07a88fdf"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:02.157305149Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1755,"":"io task","client":"0","":"downstairs","session_id":"8ce6c17b-3f87-4531-a1f6-916f07a88fdf"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:02.157319759Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1755,"":"io task","client":"1","":"downstairs","session_id":"8ce6c17b-3f87-4531-a1f6-916f07a88fdf"} {"msg":"ds_connection connected from Ok(127.0.0.1:61381)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:02.157349959Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1755,"":"io task","client":"2","":"downstairs","session_id":"8ce6c17b-3f87-4531-a1f6-916f07a88fdf"} {"msg":"ds_connection connected from Ok(127.0.0.1:58043)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:02.157366459Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1755,"":"io task","client":"0","":"downstairs","session_id":"8ce6c17b-3f87-4531-a1f6-916f07a88fdf"} {"msg":"ds_connection connected from Ok(127.0.0.1:38935)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:02.157431609Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1755,"":"io task","client":"1","":"downstairs","session_id":"8ce6c17b-3f87-4531-a1f6-916f07a88fdf"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:02.15808157Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1755,"client":"2","":"downstairs","session_id":"8ce6c17b-3f87-4531-a1f6-916f07a88fdf"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:02.15810486Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1755,"client":"0","":"downstairs","session_id":"8ce6c17b-3f87-4531-a1f6-916f07a88fdf"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:02.15813923Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1755,"client":"1","":"downstairs","session_id":"8ce6c17b-3f87-4531-a1f6-916f07a88fdf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:02.158462491Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1755,"session_id":"8ce6c17b-3f87-4531-a1f6-916f07a88fdf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:02.158474531Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1755,"session_id":"8ce6c17b-3f87-4531-a1f6-916f07a88fdf"} {"msg":"[0]R flush_numbers[0..12]: [247, 244, 254, 254, 245, 252, 246, 246, 244, 250, 249, 250]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:02.158493311Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1755,"":"downstairs","":"downstairs","session_id":"8ce6c17b-3f87-4531-a1f6-916f07a88fdf"} {"msg":"[0]R generation[0..12]: [148, 146, 153, 153, 146, 152, 147, 147, 146, 150, 149, 150]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:02.158504081Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1755,"":"downstairs","":"downstairs","session_id":"8ce6c17b-3f87-4531-a1f6-916f07a88fdf"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:02.158515271Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1755,"":"downstairs","":"downstairs","session_id":"8ce6c17b-3f87-4531-a1f6-916f07a88fdf"} {"msg":"[1]R flush_numbers[0..12]: [247, 244, 254, 254, 245, 252, 246, 246, 244, 250, 249, 250]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:02.158526181Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1755,"":"downstairs","":"downstairs","session_id":"8ce6c17b-3f87-4531-a1f6-916f07a88fdf"} {"msg":"[1]R generation[0..12]: [148, 146, 153, 153, 146, 152, 147, 147, 146, 150, 149, 150]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:02.158536841Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1755,"":"downstairs","":"downstairs","session_id":"8ce6c17b-3f87-4531-a1f6-916f07a88fdf"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:02.158548201Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1755,"":"downstairs","":"downstairs","session_id":"8ce6c17b-3f87-4531-a1f6-916f07a88fdf"} {"msg":"[2]R flush_numbers[0..12]: [247, 244, 254, 254, 245, 252, 246, 246, 244, 250, 249, 250]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:02.158559041Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1755,"":"downstairs","":"downstairs","session_id":"8ce6c17b-3f87-4531-a1f6-916f07a88fdf"} {"msg":"[2]R generation[0..12]: [148, 146, 153, 153, 146, 152, 147, 147, 146, 150, 149, 150]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:02.158571891Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1755,"":"downstairs","":"downstairs","session_id":"8ce6c17b-3f87-4531-a1f6-916f07a88fdf"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:02.158582991Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1755,"":"downstairs","":"downstairs","session_id":"8ce6c17b-3f87-4531-a1f6-916f07a88fdf"} {"msg":"Max found gen is 154","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:02.158592591Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1755,"":"downstairs","":"downstairs","session_id":"8ce6c17b-3f87-4531-a1f6-916f07a88fdf"} {"msg":"Generation requested: 154 >= found:154","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:02.158602551Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1755,"":"downstairs","":"downstairs","session_id":"8ce6c17b-3f87-4531-a1f6-916f07a88fdf"} {"msg":"Next flush: 255","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:02.158612101Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1755,"":"downstairs","":"downstairs","session_id":"8ce6c17b-3f87-4531-a1f6-916f07a88fdf"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:02.158621571Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1755,"":"downstairs","":"downstairs","session_id":"8ce6c17b-3f87-4531-a1f6-916f07a88fdf"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:02.158631621Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1755,"session_id":"8ce6c17b-3f87-4531-a1f6-916f07a88fdf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:02.158640651Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1755,"client":"0","":"downstairs","session_id":"8ce6c17b-3f87-4531-a1f6-916f07a88fdf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:02.158650701Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1755,"client":"1","":"downstairs","session_id":"8ce6c17b-3f87-4531-a1f6-916f07a88fdf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:02.158660561Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1755,"client":"2","":"downstairs","session_id":"8ce6c17b-3f87-4531-a1f6-916f07a88fdf"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:02.158670651Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1755,"session_id":"8ce6c17b-3f87-4531-a1f6-916f07a88fdf"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:02.158680121Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1755,"session_id":"8ce6c17b-3f87-4531-a1f6-916f07a88fdf"} {"msg":"d8852b41-395f-4f88-89e2-56eda2b40891 is now active with session: 8ce6c17b-3f87-4531-a1f6-916f07a88fdf","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:02.158690151Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1755,"session_id":"8ce6c17b-3f87-4531-a1f6-916f07a88fdf"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:02.158699061Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1755,"session_id":"8ce6c17b-3f87-4531-a1f6-916f07a88fdf"} {"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:02.158708371Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1755} {"msg":"Activated sub_volume d8852b41-395f-4f88-89e2-56eda2b40891","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:02.158716491Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1755} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:02.158725951Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1755,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:02.159045242Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1755,"task":"crutest"} 01/30 Write block 319 len 36864 data: 13 14 14 14 14 11 12 11 11 02/30 Write block 393 len 20480 data: 7 9 9 7 7 03/30 Read block 499 len 40960 04/30 Write block 349 len 4096 data: 12 05/30 Flush 06/30 Write block 592 len 4096 data: 7 07/30 Read block 196 len 8192 08/30 Read block 594 len 12288 09/30 Write block 500 len 8192 data: 11 11 10/30 Write block 244 len 40960 data: 12 13 12 12 12 12 11 13 9 9 11/30 Read block 279 len 32768 12/30 Read block 217 len 16384 13/30 Read block 496 len 4096 14/30 Write block 549 len 16384 data: 11 11 11 12 15/30 Read block 457 len 28672 16/30 Read block 392 len 40960 17/30 Write block 279 len 36864 data: 11 14 11 10 9 7 9 10 9 18/30 Read block 269 len 20480 19/30 Read block 328 len 32768 20/30 Read block 521 len 16384 21/30 Write block 330 len 4096 data: 10 22/30 Read block 467 len 32768 23/30 Write block 23 len 20480 data: 16 15 14 12 15 24/30 Write block 336 len 4096 data: 11 25/30 Read block 547 len 32768 26/30 Flush 27/30 Write block 591 len 32768 data: 6 8 6 6 7 5 4 3 28/30 Write block 401 len 8192 data: 6 7 29/30 Write block 215 len 36864 data: 11 12 12 13 12 9 8 8 10 30/30 Read block 462 len 20480 ---------------------------------------------------------------- Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Write 5 Done Done Done false 1002 Acked Read 10 Done Done Done false 1003 Acked Write 1 Done Done Done false 1004 Acked Flush 0 Sent Done Done false 1005 Acked Write 1 Sent Done Done false 1006 Acked Read 2 Sent Done Done false 1007 Acked Read 3 Sent Done Done false 1008 Acked Write 2 Sent Done Done false 1009 Acked Write 10 Sent Done Done false 1010 Acked Read 8 Sent Done Done false 1011 Acked Read 4 Sent Done Done false 1012 Acked Read 1 Sent Done Done false 1013 Acked Write 4 Sent Done Done false 1014 Acked Read 7 Sent Done Done false 1015 Acked Read 10 Sent Done Done false 1016 Acked Write 9 Sent Done Done false 1017 Acked Read 5 Sent Done Done false 1018 Acked Read 8 Sent Done Done false 1019 Acked Read 4 Sent Done Done false 1020 Acked Write 1 Sent Done Done false 1021 Acked Read 8 Sent Done Done false 1022 Acked Write 5 Sent Done Done false 1023 Acked Write 1 Sent Done Done false 1024 Acked Read 8 Sent Done Done false 1025 Acked Flush 0 Sent Done Done false 1026 Acked Write 8 Sent Done Done false 1027 Acked Write 2 Sent Done Done false 1028 Acked Write 9 Sent Done Done false 1029 Acked Read 5 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 26 0 0 26 Done 4 30 30 64 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:25:02.167960945Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1755} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:02.168155956Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1755,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 247 247 247 F F F 1 020-039 146 154 154 244 256 256 F F F <--- 2 040-059 153 153 153 254 254 254 F F F 3 060-079 153 153 153 254 254 254 F F F 4 080-099 146 146 146 245 245 245 F F F 5 100-119 152 152 152 252 252 252 F F F 6 120-139 147 147 147 246 246 246 F F F 7 140-159 147 147 147 246 246 246 F F F 8 160-179 146 146 146 244 244 244 F F F 9 180-199 150 150 150 250 250 250 F F F 10 200-219 149 149 149 249 249 249 F T T <--- 11 220-239 150 150 150 250 250 250 F T T <--- 12 240-259 150 154 154 250 256 256 F F F <--- 13 260-279 153 154 154 254 256 256 F F F <--- 14 280-299 153 154 154 254 256 256 F F F <--- 15 300-319 148 154 154 247 255 255 T F F <--- 16 320-339 150 154 154 250 256 256 T F F <--- 17 340-359 145 154 154 243 255 255 T F F <--- 18 360-379 148 148 148 248 248 248 F F F 19 380-399 148 154 154 247 255 255 T F F <--- 20 400-419 153 153 153 254 254 254 F T T <--- 21 420-439 153 153 153 254 254 254 F F F 22 440-459 151 151 151 251 251 251 F F F 23 460-479 153 153 153 254 254 254 F F F 24 480-499 144 144 144 242 242 242 F F F 25 500-519 148 154 154 247 256 256 F F F <--- 26 520-539 148 148 148 248 248 248 F F F 27 540-559 150 154 154 250 256 256 F F F <--- 28 560-579 153 153 153 254 254 254 F F F 29 580-599 153 154 154 254 256 256 F T T <--- Max gen: 154, Max flush: 256 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 77 Verifying data now No change after verify Loop: 77 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 247 247 247 F F F 1 020-039 154 154 154 256 256 256 F F F 2 040-059 153 153 153 254 254 254 F F F 3 060-079 153 153 153 254 254 254 F F F 4 080-099 146 146 146 245 245 245 F F F 5 100-119 152 152 152 252 252 252 F F F 6 120-139 147 147 147 246 246 246 F F F 7 140-159 147 147 147 246 246 246 F F F 8 160-179 146 146 146 244 244 244 F F F 9 180-199 150 150 150 250 250 250 F F F 10 200-219 155 155 155 257 257 257 F F F 11 220-239 155 155 155 257 257 257 F F F 12 240-259 154 154 154 256 256 256 F F F 13 260-279 154 154 154 256 256 256 F F F 14 280-299 154 154 154 256 256 256 F F F 15 300-319 154 154 154 255 255 255 F F F 16 320-339 154 154 154 256 256 256 F F F 17 340-359 154 154 154 255 255 255 F F F 18 360-379 148 148 148 248 248 248 F F F 19 380-399 154 154 154 255 255 255 F F F 20 400-419 155 155 155 257 257 257 F F F 21 420-439 153 153 153 254 254 254 F F F 22 440-459 151 151 151 251 251 251 F F F 23 460-479 153 153 153 254 254 254 F F F 24 480-499 144 144 144 242 242 242 F F F 25 500-519 154 154 154 256 256 256 F F F 26 520-539 148 148 148 248 248 248 F F F 27 540-559 154 154 154 256 256 256 F F F 28 560-579 153 153 153 254 254 254 F F F 29 580-599 155 155 155 257 257 257 F F F Max gen: 155, Max flush: 257 Begin loop 78 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:06.427380033Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1763,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:06.435714735Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1763,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:06.436015826Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1763} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:06.436042736Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1763} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:06.436051426Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1763} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:06.436063266Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1763} {"msg":"Crucible 4e5ea6a3-abc9-4056-948d-7fdd817906f7 has session id: 91729949-16eb-4604-8f2a-26acf292eaf7","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:06.436372936Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1763,"session_id":"91729949-16eb-4604-8f2a-26acf292eaf7"} {"msg":"Upstairs opts: Upstairs UUID: 4e5ea6a3-abc9-4056-948d-7fdd817906f7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:06.436396966Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1763,"session_id":"91729949-16eb-4604-8f2a-26acf292eaf7"} {"msg":"Crucible stats registered with UUID: 4e5ea6a3-abc9-4056-948d-7fdd817906f7","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:06.436406936Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1763,"session_id":"91729949-16eb-4604-8f2a-26acf292eaf7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:25:06.436416116Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1763,"session_id":"91729949-16eb-4604-8f2a-26acf292eaf7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:06.436570607Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1763,"":"io task","client":"0","":"downstairs","session_id":"91729949-16eb-4604-8f2a-26acf292eaf7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:06.436602567Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1763,"":"io task","client":"1","":"downstairs","session_id":"91729949-16eb-4604-8f2a-26acf292eaf7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:06.436615417Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1763,"":"io task","client":"2","":"downstairs","session_id":"91729949-16eb-4604-8f2a-26acf292eaf7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:06.436748747Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1763,"task":"crutest"} {"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:06.436764647Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1763} {"msg":"4e5ea6a3-abc9-4056-948d-7fdd817906f7 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:06.436788077Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1763,"session_id":"91729949-16eb-4604-8f2a-26acf292eaf7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:06.436797807Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1763,"client":"0","":"downstairs","session_id":"91729949-16eb-4604-8f2a-26acf292eaf7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:06.436807657Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1763,"client":"1","":"downstairs","session_id":"91729949-16eb-4604-8f2a-26acf292eaf7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:06.436817347Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1763,"client":"2","":"downstairs","session_id":"91729949-16eb-4604-8f2a-26acf292eaf7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:06.436827017Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1763,"":"io task","client":"2","":"downstairs","session_id":"91729949-16eb-4604-8f2a-26acf292eaf7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:06.436837037Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1763,"":"io task","client":"0","":"downstairs","session_id":"91729949-16eb-4604-8f2a-26acf292eaf7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:06.436885787Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1763,"":"io task","client":"1","":"downstairs","session_id":"91729949-16eb-4604-8f2a-26acf292eaf7"} {"msg":"ds_connection connected from Ok(127.0.0.1:39462)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:06.436910457Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1763,"":"io task","client":"2","":"downstairs","session_id":"91729949-16eb-4604-8f2a-26acf292eaf7"} {"msg":"ds_connection connected from Ok(127.0.0.1:55273)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:06.436921937Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1763,"":"io task","client":"1","":"downstairs","session_id":"91729949-16eb-4604-8f2a-26acf292eaf7"} {"msg":"ds_connection connected from Ok(127.0.0.1:45657)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:06.436932597Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1763,"":"io task","client":"0","":"downstairs","session_id":"91729949-16eb-4604-8f2a-26acf292eaf7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:06.437651988Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1763,"client":"0","":"downstairs","session_id":"91729949-16eb-4604-8f2a-26acf292eaf7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:06.437679758Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1763,"client":"1","":"downstairs","session_id":"91729949-16eb-4604-8f2a-26acf292eaf7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:06.437720098Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1763,"client":"2","":"downstairs","session_id":"91729949-16eb-4604-8f2a-26acf292eaf7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:06.437938719Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1763,"session_id":"91729949-16eb-4604-8f2a-26acf292eaf7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:06.437963879Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1763,"session_id":"91729949-16eb-4604-8f2a-26acf292eaf7"} {"msg":"[0]R flush_numbers[0..12]: [247, 256, 254, 254, 245, 252, 246, 246, 244, 250, 257, 257]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:06.438012509Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1763,"":"downstairs","":"downstairs","session_id":"91729949-16eb-4604-8f2a-26acf292eaf7"} {"msg":"[0]R generation[0..12]: [148, 154, 153, 153, 146, 152, 147, 147, 146, 150, 155, 155]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:06.438023929Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1763,"":"downstairs","":"downstairs","session_id":"91729949-16eb-4604-8f2a-26acf292eaf7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:06.438035209Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1763,"":"downstairs","":"downstairs","session_id":"91729949-16eb-4604-8f2a-26acf292eaf7"} {"msg":"[1]R flush_numbers[0..12]: [247, 256, 254, 254, 245, 252, 246, 246, 244, 250, 257, 257]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:06.438046059Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1763,"":"downstairs","":"downstairs","session_id":"91729949-16eb-4604-8f2a-26acf292eaf7"} {"msg":"[1]R generation[0..12]: [148, 154, 153, 153, 146, 152, 147, 147, 146, 150, 155, 155]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:06.438056869Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1763,"":"downstairs","":"downstairs","session_id":"91729949-16eb-4604-8f2a-26acf292eaf7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:06.438068069Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1763,"":"downstairs","":"downstairs","session_id":"91729949-16eb-4604-8f2a-26acf292eaf7"} {"msg":"[2]R flush_numbers[0..12]: [247, 256, 254, 254, 245, 252, 246, 246, 244, 250, 257, 257]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:06.438078879Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1763,"":"downstairs","":"downstairs","session_id":"91729949-16eb-4604-8f2a-26acf292eaf7"} {"msg":"[2]R generation[0..12]: [148, 154, 153, 153, 146, 152, 147, 147, 146, 150, 155, 155]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:06.438089619Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1763,"":"downstairs","":"downstairs","session_id":"91729949-16eb-4604-8f2a-26acf292eaf7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:06.438100659Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1763,"":"downstairs","":"downstairs","session_id":"91729949-16eb-4604-8f2a-26acf292eaf7"} {"msg":"Max found gen is 156","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:06.438110339Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1763,"":"downstairs","":"downstairs","session_id":"91729949-16eb-4604-8f2a-26acf292eaf7"} {"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:06.438120279Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1763,"":"downstairs","":"downstairs","session_id":"91729949-16eb-4604-8f2a-26acf292eaf7"} {"msg":"Next flush: 258","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:06.438131329Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1763,"":"downstairs","":"downstairs","session_id":"91729949-16eb-4604-8f2a-26acf292eaf7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:06.438140829Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1763,"":"downstairs","":"downstairs","session_id":"91729949-16eb-4604-8f2a-26acf292eaf7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:06.438150719Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1763,"session_id":"91729949-16eb-4604-8f2a-26acf292eaf7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:06.438159769Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1763,"client":"0","":"downstairs","session_id":"91729949-16eb-4604-8f2a-26acf292eaf7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:06.438169569Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1763,"client":"1","":"downstairs","session_id":"91729949-16eb-4604-8f2a-26acf292eaf7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:06.438179429Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1763,"client":"2","":"downstairs","session_id":"91729949-16eb-4604-8f2a-26acf292eaf7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:06.438189329Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1763,"session_id":"91729949-16eb-4604-8f2a-26acf292eaf7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:06.438198719Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1763,"session_id":"91729949-16eb-4604-8f2a-26acf292eaf7"} {"msg":"4e5ea6a3-abc9-4056-948d-7fdd817906f7 is now active with session: 91729949-16eb-4604-8f2a-26acf292eaf7","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:06.438208879Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1763,"session_id":"91729949-16eb-4604-8f2a-26acf292eaf7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:06.438217909Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1763,"session_id":"91729949-16eb-4604-8f2a-26acf292eaf7"} {"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:06.438227189Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1763} {"msg":"Activated sub_volume 4e5ea6a3-abc9-4056-948d-7fdd817906f7","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:06.438235329Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1763} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:06.438243559Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1763,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:06.43856756Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1763,"task":"crutest"} 01/30 Flush 02/30 Write block 462 len 28672 data: 13 17 15 13 12 13 14 03/30 Write block 211 len 36864 data: 13 12 11 10 12 13 13 14 13 04/30 Read block 573 len 32768 05/30 Write block 163 len 20480 data: 5 6 5 6 10 06/30 Read block 25 len 20480 07/30 Read block 358 len 4096 08/30 Read block 520 len 8192 09/30 Write block 294 len 28672 data: 14 15 15 15 17 16 14 10/30 Read block 388 len 28672 11/30 Flush 12/30 Write block 116 len 36864 data: 13 10 9 8 8 8 9 10 9 13/30 Write block 400 len 24576 data: 8 7 8 6 8 8 14/30 Read block 279 len 20480 15/30 Read block 579 len 36864 16/30 Write block 91 len 40960 data: 16 15 13 13 13 11 12 12 8 8 17/30 Read block 39 len 8192 18/30 Flush 19/30 Write block 351 len 12288 data: 5 7 9 20/30 Write block 460 len 28672 data: 14 14 14 18 16 14 13 21/30 Read block 294 len 36864 22/30 Read block 181 len 20480 23/30 Read block 235 len 36864 24/30 Read block 313 len 4096 25/30 Flush 26/30 Write block 270 len 20480 data: 17 19 18 12 14 27/30 Write block 426 len 8192 data: 8 8 28/30 Read block 216 len 40960 29/30 Write block 525 len 20480 data: 16 16 18 18 14 30/30 Write block 333 len 24576 data: 10 10 11 12 14 13 ---------------------------------------------------------------- Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 7 Done Done Done false 1002 Acked Write 9 Done Done Done false 1003 Acked Read 8 Done Sent Done false 1004 Acked Write 5 Done Sent Done false 1005 Acked Read 5 Done Sent Done false 1006 Acked Read 1 Done Sent Done false 1007 Acked Read 2 Done Sent Done false 1008 Acked Write 7 Done Sent Done false 1009 Acked Read 7 Done Sent Done false 1010 Acked Flush 0 Done Sent Done false 1011 Acked Write 9 Done Sent Done false 1012 Acked Write 6 Done Sent Done false 1013 Acked Read 5 Done Sent Done false 1014 Acked Read 9 Done Sent Done false 1015 Acked Write 10 Done Sent Done false 1016 Acked Read 2 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Write 3 Done Sent Done false 1019 Acked Write 7 Done Sent Done false 1020 Acked Read 9 Done Sent Done false 1021 Acked Read 5 Done Sent Done false 1022 Acked Read 9 Done Sent Done false 1023 Acked Read 1 Done Sent Done false 1024 Acked Flush 0 Done Sent Done false 1025 Acked Write 5 Done Sent Done false 1026 Acked Write 2 Done Sent Done false 1027 Acked Read 10 Done Sent Done false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 27 2 31 Done 27 2 27 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1000)) Some(JobId(1024)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:25:06.447890604Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1763} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:06.448052594Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1763,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 247 247 247 F F F 1 020-039 154 154 154 256 256 256 F F F 2 040-059 153 153 153 254 254 254 F F F 3 060-079 153 153 153 254 254 254 F F F 4 080-099 156 146 156 260 245 260 F F F <--- 5 100-119 156 152 156 260 252 260 F F F <--- 6 120-139 156 147 156 260 246 260 F F F <--- 7 140-159 147 147 147 246 246 246 F F F 8 160-179 156 146 156 259 244 259 F F F <--- 9 180-199 150 150 150 250 250 250 F F F 10 200-219 156 155 156 259 257 259 F T F <--- 11 220-239 155 155 155 257 257 257 F F F 12 240-259 154 154 154 256 256 256 F F F 13 260-279 154 154 154 256 256 256 T F T <--- 14 280-299 156 154 156 259 256 259 F F F <--- 15 300-319 156 154 156 259 255 259 F F F <--- 16 320-339 154 154 154 256 256 256 T F T <--- 17 340-359 156 154 156 261 255 261 F F F <--- 18 360-379 148 148 148 248 248 248 F F F 19 380-399 154 154 154 255 255 255 F F F 20 400-419 156 155 156 260 257 260 F F F <--- 21 420-439 153 153 153 254 254 254 T F T <--- 22 440-459 151 151 151 251 251 251 F F F 23 460-479 156 153 156 261 254 261 F T F <--- 24 480-499 144 144 144 242 242 242 F F F 25 500-519 154 154 154 256 256 256 F F F 26 520-539 148 148 148 248 248 248 T F T <--- 27 540-559 154 154 154 256 256 256 F F F 28 560-579 153 153 153 254 254 254 F F F 29 580-599 155 155 155 257 257 257 F F F Max gen: 156, Max flush: 261 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 78 Verifying data now No change after verify Loop: 78 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 247 247 247 F F F 1 020-039 154 154 154 256 256 256 F F F 2 040-059 153 153 153 254 254 254 F F F 3 060-079 153 153 153 254 254 254 F F F 4 080-099 156 156 156 260 260 260 F F F 5 100-119 156 156 156 260 260 260 F F F 6 120-139 156 156 156 260 260 260 F F F 7 140-159 147 147 147 246 246 246 F F F 8 160-179 156 156 156 259 259 259 F F F 9 180-199 150 150 150 250 250 250 F F F 10 200-219 156 156 156 259 259 259 F F F 11 220-239 155 155 155 257 257 257 F F F 12 240-259 154 154 154 256 256 256 F F F 13 260-279 157 157 157 262 262 262 F F F 14 280-299 156 156 156 259 259 259 F F F 15 300-319 156 156 156 259 259 259 F F F 16 320-339 157 157 157 262 262 262 F F F 17 340-359 156 156 156 261 261 261 F F F 18 360-379 148 148 148 248 248 248 F F F 19 380-399 154 154 154 255 255 255 F F F 20 400-419 156 156 156 260 260 260 F F F 21 420-439 157 157 157 262 262 262 F F F 22 440-459 151 151 151 251 251 251 F F F 23 460-479 156 156 156 261 261 261 F F F 24 480-499 144 144 144 242 242 242 F F F 25 500-519 154 154 154 256 256 256 F F F 26 520-539 157 157 157 262 262 262 F F F 27 540-559 154 154 154 256 256 256 F F F 28 560-579 153 153 153 254 254 254 F F F 29 580-599 155 155 155 257 257 257 F F F Max gen: 157, Max flush: 262 Begin loop 79 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:10.703117095Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1771,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:10.711379598Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1771,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:10.711681168Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1771} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:10.711708448Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1771} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:10.711719598Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1771} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:10.711731558Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1771} {"msg":"Crucible 17f328b3-c1aa-4e3f-9363-2dbfc6362d9e has session id: 535e8ac2-366e-47f5-9193-f9ee00327f4e","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:10.712025199Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1771,"session_id":"535e8ac2-366e-47f5-9193-f9ee00327f4e"} {"msg":"Upstairs opts: Upstairs UUID: 17f328b3-c1aa-4e3f-9363-2dbfc6362d9e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:10.712056309Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1771,"session_id":"535e8ac2-366e-47f5-9193-f9ee00327f4e"} {"msg":"Crucible stats registered with UUID: 17f328b3-c1aa-4e3f-9363-2dbfc6362d9e","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:10.712066709Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1771,"session_id":"535e8ac2-366e-47f5-9193-f9ee00327f4e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:25:10.712075819Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1771,"session_id":"535e8ac2-366e-47f5-9193-f9ee00327f4e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:10.712229419Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1771,"":"io task","client":"1","":"downstairs","session_id":"535e8ac2-366e-47f5-9193-f9ee00327f4e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:10.712260649Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1771,"":"io task","client":"0","":"downstairs","session_id":"535e8ac2-366e-47f5-9193-f9ee00327f4e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:10.712271719Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1771,"":"io task","client":"2","":"downstairs","session_id":"535e8ac2-366e-47f5-9193-f9ee00327f4e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:10.712403699Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1771,"task":"crutest"} {"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:10.712427759Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1771} {"msg":"17f328b3-c1aa-4e3f-9363-2dbfc6362d9e active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:10.712454249Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1771,"session_id":"535e8ac2-366e-47f5-9193-f9ee00327f4e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:10.712463649Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1771,"client":"0","":"downstairs","session_id":"535e8ac2-366e-47f5-9193-f9ee00327f4e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:10.712473249Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1771,"client":"1","":"downstairs","session_id":"535e8ac2-366e-47f5-9193-f9ee00327f4e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:10.712485149Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1771,"client":"2","":"downstairs","session_id":"535e8ac2-366e-47f5-9193-f9ee00327f4e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:10.712494859Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1771,"":"io task","client":"2","":"downstairs","session_id":"535e8ac2-366e-47f5-9193-f9ee00327f4e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:10.712504929Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1771,"":"io task","client":"0","":"downstairs","session_id":"535e8ac2-366e-47f5-9193-f9ee00327f4e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:10.712552669Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1771,"":"io task","client":"1","":"downstairs","session_id":"535e8ac2-366e-47f5-9193-f9ee00327f4e"} {"msg":"ds_connection connected from Ok(127.0.0.1:34353)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:10.712582969Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1771,"":"io task","client":"2","":"downstairs","session_id":"535e8ac2-366e-47f5-9193-f9ee00327f4e"} {"msg":"ds_connection connected from Ok(127.0.0.1:52288)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:10.712597459Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1771,"":"io task","client":"1","":"downstairs","session_id":"535e8ac2-366e-47f5-9193-f9ee00327f4e"} {"msg":"ds_connection connected from Ok(127.0.0.1:35029)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:10.712608099Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1771,"":"io task","client":"0","":"downstairs","session_id":"535e8ac2-366e-47f5-9193-f9ee00327f4e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:10.713361251Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1771,"client":"1","":"downstairs","session_id":"535e8ac2-366e-47f5-9193-f9ee00327f4e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:10.713379041Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1771,"client":"0","":"downstairs","session_id":"535e8ac2-366e-47f5-9193-f9ee00327f4e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:10.713448941Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1771,"client":"2","":"downstairs","session_id":"535e8ac2-366e-47f5-9193-f9ee00327f4e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:10.713697841Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1771,"session_id":"535e8ac2-366e-47f5-9193-f9ee00327f4e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:10.713732181Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1771,"session_id":"535e8ac2-366e-47f5-9193-f9ee00327f4e"} {"msg":"[0]R flush_numbers[0..12]: [247, 256, 254, 254, 260, 260, 260, 246, 259, 250, 259, 257]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:10.713817061Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1771,"":"downstairs","":"downstairs","session_id":"535e8ac2-366e-47f5-9193-f9ee00327f4e"} {"msg":"[0]R generation[0..12]: [148, 154, 153, 153, 156, 156, 156, 147, 156, 150, 156, 155]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:10.713828481Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1771,"":"downstairs","":"downstairs","session_id":"535e8ac2-366e-47f5-9193-f9ee00327f4e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:10.713839561Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1771,"":"downstairs","":"downstairs","session_id":"535e8ac2-366e-47f5-9193-f9ee00327f4e"} {"msg":"[1]R flush_numbers[0..12]: [247, 256, 254, 254, 260, 260, 260, 246, 259, 250, 259, 257]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:10.713852081Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1771,"":"downstairs","":"downstairs","session_id":"535e8ac2-366e-47f5-9193-f9ee00327f4e"} {"msg":"[1]R generation[0..12]: [148, 154, 153, 153, 156, 156, 156, 147, 156, 150, 156, 155]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:10.713862871Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1771,"":"downstairs","":"downstairs","session_id":"535e8ac2-366e-47f5-9193-f9ee00327f4e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:10.713873941Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1771,"":"downstairs","":"downstairs","session_id":"535e8ac2-366e-47f5-9193-f9ee00327f4e"} {"msg":"[2]R flush_numbers[0..12]: [247, 256, 254, 254, 260, 260, 260, 246, 259, 250, 259, 257]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:10.713884731Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1771,"":"downstairs","":"downstairs","session_id":"535e8ac2-366e-47f5-9193-f9ee00327f4e"} {"msg":"[2]R generation[0..12]: [148, 154, 153, 153, 156, 156, 156, 147, 156, 150, 156, 155]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:10.713895291Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1771,"":"downstairs","":"downstairs","session_id":"535e8ac2-366e-47f5-9193-f9ee00327f4e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:10.713906311Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1771,"":"downstairs","":"downstairs","session_id":"535e8ac2-366e-47f5-9193-f9ee00327f4e"} {"msg":"Max found gen is 158","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:10.713915891Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1771,"":"downstairs","":"downstairs","session_id":"535e8ac2-366e-47f5-9193-f9ee00327f4e"} {"msg":"Generation requested: 158 >= found:158","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:10.713925731Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1771,"":"downstairs","":"downstairs","session_id":"535e8ac2-366e-47f5-9193-f9ee00327f4e"} {"msg":"Next flush: 263","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:10.713935271Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1771,"":"downstairs","":"downstairs","session_id":"535e8ac2-366e-47f5-9193-f9ee00327f4e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:10.713944831Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1771,"":"downstairs","":"downstairs","session_id":"535e8ac2-366e-47f5-9193-f9ee00327f4e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:10.713954571Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1771,"session_id":"535e8ac2-366e-47f5-9193-f9ee00327f4e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:10.713963501Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1771,"client":"0","":"downstairs","session_id":"535e8ac2-366e-47f5-9193-f9ee00327f4e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:10.713973251Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1771,"client":"1","":"downstairs","session_id":"535e8ac2-366e-47f5-9193-f9ee00327f4e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:10.713983121Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1771,"client":"2","":"downstairs","session_id":"535e8ac2-366e-47f5-9193-f9ee00327f4e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:10.713994091Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1771,"session_id":"535e8ac2-366e-47f5-9193-f9ee00327f4e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:10.714003361Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1771,"session_id":"535e8ac2-366e-47f5-9193-f9ee00327f4e"} {"msg":"17f328b3-c1aa-4e3f-9363-2dbfc6362d9e is now active with session: 535e8ac2-366e-47f5-9193-f9ee00327f4e","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:10.714013532Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1771,"session_id":"535e8ac2-366e-47f5-9193-f9ee00327f4e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:10.714022302Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1771,"session_id":"535e8ac2-366e-47f5-9193-f9ee00327f4e"} {"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:10.714031482Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1771} {"msg":"Activated sub_volume 17f328b3-c1aa-4e3f-9363-2dbfc6362d9e","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:10.714039562Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1771} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:10.714047782Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1771,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:10.714405602Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1771,"task":"crutest"} 01/30 Write block 258 len 4096 data: 11 02/30 Flush 03/30 Write block 356 len 4096 data: 10 04/30 Read block 19 len 4096 05/30 Write block 437 len 28672 data: 9 10 10 8 7 9 9 06/30 Write block 120 len 12288 data: 9 9 10 07/30 Read block 515 len 12288 08/30 Read block 60 len 24576 09/30 Read block 420 len 36864 10/30 Flush 11/30 Write block 332 len 16384 data: 13 11 11 12 12/30 Write block 5 len 8192 data: 10 12 13/30 Write block 316 len 36864 data: 9 11 12 14 15 15 15 15 12 14/30 Read block 451 len 20480 15/30 Read block 159 len 12288 16/30 Write block 90 len 36864 data: 16 17 16 14 14 14 12 13 13 17/30 Write block 507 len 12288 data: 9 9 9 18/30 Read block 124 len 32768 19/30 Read block 365 len 4096 20/30 Write block 409 len 8192 data: 11 12 21/30 Read block 500 len 4096 22/30 Read block 242 len 20480 23/30 Read block 150 len 40960 24/30 Flush 25/30 Write block 237 len 20480 data: 15 14 16 16 17 26/30 Read block 78 len 4096 27/30 Write block 514 len 16384 data: 10 9 8 9 28/30 Write block 488 len 8192 data: 17 17 29/30 Write block 157 len 36864 data: 12 15 13 13 12 7 6 7 6 30/30 Write block 153 len 24576 data: 11 14 13 13 13 16 ---------------------------------------------------------------- Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Sent false 1001 Acked Flush 0 Done Done Sent false 1002 Acked Write 1 Done Done Sent false 1003 Acked Read 1 Done Done Sent false 1004 Acked Write 7 Done Done Sent false 1005 Acked Write 3 Done Done Sent false 1006 Acked Read 3 Done Done Sent false 1007 Acked Read 6 Done Done Sent false 1008 Acked Read 9 Done Done Sent false 1009 Acked Flush 0 Done Done Sent false 1010 Acked Write 4 Done Done Sent false 1011 Acked Write 2 Done Done Sent false 1012 Acked Write 9 Done Done Sent false 1013 Acked Read 5 Done Done Sent false 1014 Acked Read 3 Done Done Sent false 1015 Acked Write 9 Done Done Sent false 1016 Acked Write 3 Done Done Sent false 1017 Acked Read 8 Done Done Sent false 1018 Acked Read 1 Done Done Sent false 1019 Acked Write 2 Done Done Sent false 1020 Acked Read 1 Done Done Sent false 1021 Acked Read 5 Done Done Sent false 1022 Acked Read 10 Done Done Sent false 1023 Acked Flush 0 Done Done Sent false 1024 Acked Write 5 Done Done Sent false 1025 Acked Read 1 Done Done Sent false 1026 Acked Write 4 Done Done Sent false 1027 Acked Write 2 Done Done Sent false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 30 34 Done 28 28 0 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:25:10.722707705Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1771} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:10.722919885Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1771,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 148 265 265 247 F F F <--- 1 020-039 154 154 154 256 256 256 F F F 2 040-059 153 153 153 254 254 254 F F F 3 060-079 153 153 153 254 254 254 F F F 4 080-099 158 158 156 265 265 260 F F F <--- 5 100-119 156 156 156 260 260 260 F F F 6 120-139 158 158 156 264 264 260 F F F <--- 7 140-159 147 147 147 246 246 246 T T F <--- 8 160-179 156 156 156 259 259 259 T T F <--- 9 180-199 150 150 150 250 250 250 F F F 10 200-219 156 156 156 259 259 259 F F F 11 220-239 155 155 155 257 257 257 T T F <--- 12 240-259 158 158 154 263 263 256 T T F <--- 13 260-279 157 157 157 262 262 262 F F F 14 280-299 156 156 156 259 259 259 F F F 15 300-319 158 158 156 265 265 259 F F F <--- 16 320-339 158 158 157 265 265 262 F F F <--- 17 340-359 158 158 156 264 264 261 F F F <--- 18 360-379 148 148 148 248 248 248 F F F 19 380-399 154 154 154 255 255 255 F F F 20 400-419 158 158 156 265 265 260 F F F <--- 21 420-439 158 158 157 264 264 262 F F F <--- 22 440-459 158 158 151 264 264 251 F F F <--- 23 460-479 156 156 156 261 261 261 F F F 24 480-499 144 144 144 242 242 242 T T F <--- 25 500-519 158 158 154 265 265 256 T T F <--- 26 520-539 157 157 157 262 262 262 F F F 27 540-559 154 154 154 256 256 256 F F F 28 560-579 153 153 153 254 254 254 F F F 29 580-599 155 155 155 257 257 257 F F F Max gen: 158, Max flush: 265 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 79 Verifying data now No change after verify Loop: 79 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 265 265 265 F F F 1 020-039 154 154 154 256 256 256 F F F 2 040-059 153 153 153 254 254 254 F F F 3 060-079 153 153 153 254 254 254 F F F 4 080-099 158 158 158 265 265 265 F F F 5 100-119 156 156 156 260 260 260 F F F 6 120-139 158 158 158 264 264 264 F F F 7 140-159 159 159 159 266 266 266 F F F 8 160-179 159 159 159 266 266 266 F F F 9 180-199 150 150 150 250 250 250 F F F 10 200-219 156 156 156 259 259 259 F F F 11 220-239 159 159 159 266 266 266 F F F 12 240-259 159 159 159 266 266 266 F F F 13 260-279 157 157 157 262 262 262 F F F 14 280-299 156 156 156 259 259 259 F F F 15 300-319 158 158 158 265 265 265 F F F 16 320-339 158 158 158 265 265 265 F F F 17 340-359 158 158 158 264 264 264 F F F 18 360-379 148 148 148 248 248 248 F F F 19 380-399 154 154 154 255 255 255 F F F 20 400-419 158 158 158 265 265 265 F F F 21 420-439 158 158 158 264 264 264 F F F 22 440-459 158 158 158 264 264 264 F F F 23 460-479 156 156 156 261 261 261 F F F 24 480-499 159 159 159 266 266 266 F F F 25 500-519 159 159 159 266 266 266 F F F 26 520-539 157 157 157 262 262 262 F F F 27 540-559 154 154 154 256 256 256 F F F 28 560-579 153 153 153 254 254 254 F F F 29 580-599 155 155 155 257 257 257 F F F Max gen: 159, Max flush: 266 Begin loop 80 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:14.980328999Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1779,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:25:14.98747461Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1779,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:14.98752566Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1779,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:14.99453087Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1779,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:14.994797771Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1779} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:14.994828561Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1779} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:14.994837311Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1779} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:14.994849221Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1779} {"msg":"Crucible cb082d76-accf-43de-99ad-63b4d7a44a32 has session id: 192c842a-59e5-4c2a-b875-a3589706cfdf","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:14.995203241Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1779,"session_id":"192c842a-59e5-4c2a-b875-a3589706cfdf"} {"msg":"Upstairs opts: Upstairs UUID: cb082d76-accf-43de-99ad-63b4d7a44a32, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:14.995232622Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1779,"session_id":"192c842a-59e5-4c2a-b875-a3589706cfdf"} {"msg":"Crucible stats registered with UUID: cb082d76-accf-43de-99ad-63b4d7a44a32","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:14.995243962Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1779,"session_id":"192c842a-59e5-4c2a-b875-a3589706cfdf"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:25:14.995253382Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1779,"session_id":"192c842a-59e5-4c2a-b875-a3589706cfdf"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:14.995372982Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1779,"":"io task","client":"1","":"downstairs","session_id":"192c842a-59e5-4c2a-b875-a3589706cfdf"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:14.995392752Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1779,"":"io task","client":"0","":"downstairs","session_id":"192c842a-59e5-4c2a-b875-a3589706cfdf"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:14.995403012Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1779,"":"io task","client":"2","":"downstairs","session_id":"192c842a-59e5-4c2a-b875-a3589706cfdf"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:14.995562162Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1779,"task":"crutest"} {"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:14.995581222Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1779} {"msg":"cb082d76-accf-43de-99ad-63b4d7a44a32 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:14.995596162Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1779,"session_id":"192c842a-59e5-4c2a-b875-a3589706cfdf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:14.995605102Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1779,"client":"0","":"downstairs","session_id":"192c842a-59e5-4c2a-b875-a3589706cfdf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:14.995615072Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1779,"client":"1","":"downstairs","session_id":"192c842a-59e5-4c2a-b875-a3589706cfdf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:14.995624992Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1779,"client":"2","":"downstairs","session_id":"192c842a-59e5-4c2a-b875-a3589706cfdf"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:14.995634762Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1779,"":"io task","client":"2","":"downstairs","session_id":"192c842a-59e5-4c2a-b875-a3589706cfdf"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:14.995644982Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1779,"":"io task","client":"0","":"downstairs","session_id":"192c842a-59e5-4c2a-b875-a3589706cfdf"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:14.995656672Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1779,"":"io task","client":"1","":"downstairs","session_id":"192c842a-59e5-4c2a-b875-a3589706cfdf"} {"msg":"ds_connection connected from Ok(127.0.0.1:39118)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:14.995699832Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1779,"":"io task","client":"2","":"downstairs","session_id":"192c842a-59e5-4c2a-b875-a3589706cfdf"} {"msg":"ds_connection connected from Ok(127.0.0.1:40288)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:14.995710992Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1779,"":"io task","client":"0","":"downstairs","session_id":"192c842a-59e5-4c2a-b875-a3589706cfdf"} {"msg":"ds_connection connected from Ok(127.0.0.1:45180)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:14.995721512Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1779,"":"io task","client":"1","":"downstairs","session_id":"192c842a-59e5-4c2a-b875-a3589706cfdf"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:14.996466583Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1779,"client":"1","":"downstairs","session_id":"192c842a-59e5-4c2a-b875-a3589706cfdf"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:14.996488933Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1779,"client":"2","":"downstairs","session_id":"192c842a-59e5-4c2a-b875-a3589706cfdf"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:14.996500213Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1779,"client":"0","":"downstairs","session_id":"192c842a-59e5-4c2a-b875-a3589706cfdf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:14.996816754Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1779,"session_id":"192c842a-59e5-4c2a-b875-a3589706cfdf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:14.996849854Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1779,"session_id":"192c842a-59e5-4c2a-b875-a3589706cfdf"} {"msg":"[0]R flush_numbers[0..12]: [265, 256, 254, 254, 265, 260, 264, 266, 266, 250, 259, 266]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:14.996905924Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1779,"":"downstairs","":"downstairs","session_id":"192c842a-59e5-4c2a-b875-a3589706cfdf"} {"msg":"[0]R generation[0..12]: [158, 154, 153, 153, 158, 156, 158, 159, 159, 150, 156, 159]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:14.996917324Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1779,"":"downstairs","":"downstairs","session_id":"192c842a-59e5-4c2a-b875-a3589706cfdf"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:14.996928624Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1779,"":"downstairs","":"downstairs","session_id":"192c842a-59e5-4c2a-b875-a3589706cfdf"} {"msg":"[1]R flush_numbers[0..12]: [265, 256, 254, 254, 265, 260, 264, 266, 266, 250, 259, 266]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:14.996939304Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1779,"":"downstairs","":"downstairs","session_id":"192c842a-59e5-4c2a-b875-a3589706cfdf"} {"msg":"[1]R generation[0..12]: [158, 154, 153, 153, 158, 156, 158, 159, 159, 150, 156, 159]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:14.996949934Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1779,"":"downstairs","":"downstairs","session_id":"192c842a-59e5-4c2a-b875-a3589706cfdf"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:14.996963334Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1779,"":"downstairs","":"downstairs","session_id":"192c842a-59e5-4c2a-b875-a3589706cfdf"} {"msg":"[2]R flush_numbers[0..12]: [265, 256, 254, 254, 265, 260, 264, 266, 266, 250, 259, 266]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:14.996974134Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1779,"":"downstairs","":"downstairs","session_id":"192c842a-59e5-4c2a-b875-a3589706cfdf"} {"msg":"[2]R generation[0..12]: [158, 154, 153, 153, 158, 156, 158, 159, 159, 150, 156, 159]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:14.996984884Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1779,"":"downstairs","":"downstairs","session_id":"192c842a-59e5-4c2a-b875-a3589706cfdf"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:14.996996144Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1779,"":"downstairs","":"downstairs","session_id":"192c842a-59e5-4c2a-b875-a3589706cfdf"} {"msg":"Max found gen is 160","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:14.997005834Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1779,"":"downstairs","":"downstairs","session_id":"192c842a-59e5-4c2a-b875-a3589706cfdf"} {"msg":"Generation requested: 160 >= found:160","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:14.997016044Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1779,"":"downstairs","":"downstairs","session_id":"192c842a-59e5-4c2a-b875-a3589706cfdf"} {"msg":"Next flush: 267","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:14.997025584Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1779,"":"downstairs","":"downstairs","session_id":"192c842a-59e5-4c2a-b875-a3589706cfdf"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:14.997035084Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1779,"":"downstairs","":"downstairs","session_id":"192c842a-59e5-4c2a-b875-a3589706cfdf"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:14.997045214Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1779,"session_id":"192c842a-59e5-4c2a-b875-a3589706cfdf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:14.997054284Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1779,"client":"0","":"downstairs","session_id":"192c842a-59e5-4c2a-b875-a3589706cfdf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:14.997064414Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1779,"client":"1","":"downstairs","session_id":"192c842a-59e5-4c2a-b875-a3589706cfdf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:14.997074414Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1779,"client":"2","":"downstairs","session_id":"192c842a-59e5-4c2a-b875-a3589706cfdf"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:14.997084494Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1779,"session_id":"192c842a-59e5-4c2a-b875-a3589706cfdf"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:14.997093764Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1779,"session_id":"192c842a-59e5-4c2a-b875-a3589706cfdf"} {"msg":"cb082d76-accf-43de-99ad-63b4d7a44a32 is now active with session: 192c842a-59e5-4c2a-b875-a3589706cfdf","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:14.997103894Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1779,"session_id":"192c842a-59e5-4c2a-b875-a3589706cfdf"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:14.997112944Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1779,"session_id":"192c842a-59e5-4c2a-b875-a3589706cfdf"} {"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:14.997123954Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1779} {"msg":"Activated sub_volume cb082d76-accf-43de-99ad-63b4d7a44a32","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:14.997132234Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1779} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:14.997140514Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1779,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:14.997413285Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1779,"task":"crutest"} 01/30 Write block 567 len 36864 data: 15 13 15 15 15 13 14 15 16 02/30 Write block 183 len 20480 data: 9 11 15 15 17 03/30 Write block 251 len 8192 data: 14 10 04/30 Read block 469 len 8192 05/30 Read block 192 len 32768 06/30 Write block 83 len 32768 data: 15 14 16 15 14 15 17 17 07/30 Write block 543 len 12288 data: 14 17 13 08/30 Read block 266 len 36864 09/30 Flush 10/30 Write block 590 len 40960 data: 7 7 9 7 7 8 6 5 4 2 11/30 Flush 12/30 Write block 381 len 4096 data: 15 13/30 Write block 269 len 36864 data: 18 18 20 19 13 15 17 16 16 14/30 Flush 15/30 Write block 261 len 12288 data: 13 12 14 16/30 Read block 173 len 4096 17/30 Flush 18/30 Write block 101 len 16384 data: 6 7 9 12 19/30 Read block 375 len 32768 20/30 Read block 348 len 4096 21/30 Read block 375 len 36864 22/30 Read block 426 len 36864 23/30 Read block 434 len 4096 24/30 Read block 228 len 24576 25/30 Write block 420 len 8192 data: 10 12 26/30 Write block 552 len 4096 data: 13 27/30 Read block 111 len 32768 28/30 Read block 345 len 20480 29/30 Read block 309 len 28672 30/30 Write block 390 len 32768 data: 10 10 11 8 10 10 8 8 ---------------------------------------------------------------- Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Write 5 Done Done Done false 1002 Acked Write 2 Done Done Done false 1003 Acked Read 2 Done Done Done false 1004 Acked Read 8 Sent Done Done false 1005 Acked Write 8 Sent Done Done false 1006 Acked Write 3 Sent Done Done false 1007 Acked Read 9 Sent Done Done false 1008 Acked Flush 0 Sent Done Done false 1009 Acked Write 10 Sent Done Done false 1010 Acked Flush 0 Sent Done Done false 1011 Acked Write 1 Sent Done Done false 1012 Acked Write 9 Sent Done Done false 1013 Acked Flush 0 Sent Done Done false 1014 Acked Write 3 Sent Done Done false 1015 Acked Read 1 Sent Done Done false 1016 Acked Flush 0 Sent Done Done false 1017 Acked Write 4 Sent Done Done false 1018 Acked Read 8 Sent Done Done false 1019 Acked Read 1 Sent Done Done false 1020 Acked Read 9 Sent Done Done false 1021 Acked Read 9 Sent Done Done false 1022 Acked Read 1 Sent Done Done false 1023 Acked Read 6 Sent Done Done false 1024 Acked Write 2 Sent Done Done false 1025 Acked Write 1 Sent Done Done false 1026 Acked Read 8 Sent Done Done false 1027 Acked Read 5 Sent Done Done false 1028 Acked Read 7 Sent Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 26 1 1 28 Done 4 29 29 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:25:15.006107698Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1779} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:15.006321108Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1779,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 265 265 265 F F F 1 020-039 154 154 154 256 256 256 F F F 2 040-059 153 153 153 254 254 254 F F F 3 060-079 153 153 153 254 254 254 F F F 4 080-099 158 160 160 265 267 267 F F F <--- 5 100-119 156 156 156 260 260 260 F T T <--- 6 120-139 158 158 158 264 264 264 F F F 7 140-159 159 159 159 266 266 266 F F F 8 160-179 159 159 159 266 266 266 F F F 9 180-199 150 160 160 250 267 267 T F F <--- 10 200-219 156 156 156 259 259 259 F F F 11 220-239 159 159 159 266 266 266 F F F 12 240-259 159 160 160 266 267 267 T F F <--- 13 260-279 157 160 160 262 270 270 F F F <--- 14 280-299 156 156 156 259 259 259 F F F 15 300-319 158 158 158 265 265 265 F F F 16 320-339 158 158 158 265 265 265 F F F 17 340-359 158 158 158 264 264 264 F F F 18 360-379 148 148 148 248 248 248 F F F 19 380-399 154 160 160 255 269 269 F T T <--- 20 400-419 158 158 158 265 265 265 F F F 21 420-439 158 158 158 264 264 264 F T T <--- 22 440-459 158 158 158 264 264 264 F F F 23 460-479 156 156 156 261 261 261 F F F 24 480-499 159 159 159 266 266 266 F F F 25 500-519 159 159 159 266 266 266 F F F 26 520-539 157 157 157 262 262 262 F F F 27 540-559 154 160 160 256 267 267 F T T <--- 28 560-579 153 160 160 254 267 267 T F F <--- 29 580-599 155 160 160 257 268 268 F F F <--- Max gen: 160, Max flush: 270 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 80 Verifying data now No change after verify Loop: 80 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 265 265 265 F F F 1 020-039 154 154 154 256 256 256 F F F 2 040-059 153 153 153 254 254 254 F F F 3 060-079 153 153 153 254 254 254 F F F 4 080-099 160 160 160 267 267 267 F F F 5 100-119 161 161 161 271 271 271 F F F 6 120-139 158 158 158 264 264 264 F F F 7 140-159 159 159 159 266 266 266 F F F 8 160-179 159 159 159 266 266 266 F F F 9 180-199 160 160 160 267 267 267 F F F 10 200-219 156 156 156 259 259 259 F F F 11 220-239 159 159 159 266 266 266 F F F 12 240-259 160 160 160 267 267 267 F F F 13 260-279 160 160 160 270 270 270 F F F 14 280-299 156 156 156 259 259 259 F F F 15 300-319 158 158 158 265 265 265 F F F 16 320-339 158 158 158 265 265 265 F F F 17 340-359 158 158 158 264 264 264 F F F 18 360-379 148 148 148 248 248 248 F F F 19 380-399 161 161 161 271 271 271 F F F 20 400-419 158 158 158 265 265 265 F F F 21 420-439 161 161 161 271 271 271 F F F 22 440-459 158 158 158 264 264 264 F F F 23 460-479 156 156 156 261 261 261 F F F 24 480-499 159 159 159 266 266 266 F F F 25 500-519 159 159 159 266 266 266 F F F 26 520-539 157 157 157 262 262 262 F F F 27 540-559 161 161 161 271 271 271 F F F 28 560-579 160 160 160 267 267 267 F F F 29 580-599 160 160 160 268 268 268 F F F Max gen: 161, Max flush: 271 Begin loop 81 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:19.258710105Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1787,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:19.266959648Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1787,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:19.267250118Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1787} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:19.267278338Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1787} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:19.267287228Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1787} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:19.267298948Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1787} {"msg":"Crucible e2ebc6a0-b9b1-4edd-8b35-3de4ad84dec4 has session id: 0d27449f-288f-4651-9ec2-be17ddc0a2d0","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:19.267638849Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1787,"session_id":"0d27449f-288f-4651-9ec2-be17ddc0a2d0"} {"msg":"Upstairs opts: Upstairs UUID: e2ebc6a0-b9b1-4edd-8b35-3de4ad84dec4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:19.267661169Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1787,"session_id":"0d27449f-288f-4651-9ec2-be17ddc0a2d0"} {"msg":"Crucible stats registered with UUID: e2ebc6a0-b9b1-4edd-8b35-3de4ad84dec4","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:19.267671479Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1787,"session_id":"0d27449f-288f-4651-9ec2-be17ddc0a2d0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:25:19.267680829Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1787,"session_id":"0d27449f-288f-4651-9ec2-be17ddc0a2d0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:19.267828429Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1787,"":"io task","client":"2","":"downstairs","session_id":"0d27449f-288f-4651-9ec2-be17ddc0a2d0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:19.267846699Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1787,"":"io task","client":"0","":"downstairs","session_id":"0d27449f-288f-4651-9ec2-be17ddc0a2d0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:19.267856829Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1787,"":"io task","client":"1","":"downstairs","session_id":"0d27449f-288f-4651-9ec2-be17ddc0a2d0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:19.268033039Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1787,"task":"crutest"} {"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:19.268049619Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1787} {"msg":"e2ebc6a0-b9b1-4edd-8b35-3de4ad84dec4 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:19.268068329Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1787,"session_id":"0d27449f-288f-4651-9ec2-be17ddc0a2d0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:19.268077319Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1787,"client":"0","":"downstairs","session_id":"0d27449f-288f-4651-9ec2-be17ddc0a2d0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:19.268087089Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1787,"client":"1","":"downstairs","session_id":"0d27449f-288f-4651-9ec2-be17ddc0a2d0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:19.268096789Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1787,"client":"2","":"downstairs","session_id":"0d27449f-288f-4651-9ec2-be17ddc0a2d0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:19.268106409Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1787,"":"io task","client":"2","":"downstairs","session_id":"0d27449f-288f-4651-9ec2-be17ddc0a2d0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:19.268116309Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1787,"":"io task","client":"0","":"downstairs","session_id":"0d27449f-288f-4651-9ec2-be17ddc0a2d0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:19.268126429Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1787,"":"io task","client":"1","":"downstairs","session_id":"0d27449f-288f-4651-9ec2-be17ddc0a2d0"} {"msg":"ds_connection connected from Ok(127.0.0.1:58939)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:19.26819582Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1787,"":"io task","client":"1","":"downstairs","session_id":"0d27449f-288f-4651-9ec2-be17ddc0a2d0"} {"msg":"ds_connection connected from Ok(127.0.0.1:43918)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:19.26822185Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1787,"":"io task","client":"2","":"downstairs","session_id":"0d27449f-288f-4651-9ec2-be17ddc0a2d0"} {"msg":"ds_connection connected from Ok(127.0.0.1:52493)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:19.26823316Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1787,"":"io task","client":"0","":"downstairs","session_id":"0d27449f-288f-4651-9ec2-be17ddc0a2d0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:19.268854881Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1787,"client":"0","":"downstairs","session_id":"0d27449f-288f-4651-9ec2-be17ddc0a2d0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:19.268875901Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1787,"client":"1","":"downstairs","session_id":"0d27449f-288f-4651-9ec2-be17ddc0a2d0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:19.268899181Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1787,"client":"2","":"downstairs","session_id":"0d27449f-288f-4651-9ec2-be17ddc0a2d0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:19.269181861Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1787,"session_id":"0d27449f-288f-4651-9ec2-be17ddc0a2d0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:19.269224601Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1787,"session_id":"0d27449f-288f-4651-9ec2-be17ddc0a2d0"} {"msg":"[0]R flush_numbers[0..12]: [265, 256, 254, 254, 267, 271, 264, 266, 266, 267, 259, 266]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:19.269284871Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1787,"":"downstairs","":"downstairs","session_id":"0d27449f-288f-4651-9ec2-be17ddc0a2d0"} {"msg":"[0]R generation[0..12]: [158, 154, 153, 153, 160, 161, 158, 159, 159, 160, 156, 159]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:19.269296301Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1787,"":"downstairs","":"downstairs","session_id":"0d27449f-288f-4651-9ec2-be17ddc0a2d0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:19.269307361Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1787,"":"downstairs","":"downstairs","session_id":"0d27449f-288f-4651-9ec2-be17ddc0a2d0"} {"msg":"[1]R flush_numbers[0..12]: [265, 256, 254, 254, 267, 271, 264, 266, 266, 267, 259, 266]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:19.269318121Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1787,"":"downstairs","":"downstairs","session_id":"0d27449f-288f-4651-9ec2-be17ddc0a2d0"} {"msg":"[1]R generation[0..12]: [158, 154, 153, 153, 160, 161, 158, 159, 159, 160, 156, 159]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:19.269328711Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1787,"":"downstairs","":"downstairs","session_id":"0d27449f-288f-4651-9ec2-be17ddc0a2d0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:19.269339661Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1787,"":"downstairs","":"downstairs","session_id":"0d27449f-288f-4651-9ec2-be17ddc0a2d0"} {"msg":"[2]R flush_numbers[0..12]: [265, 256, 254, 254, 267, 271, 264, 266, 266, 267, 259, 266]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:19.269350411Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1787,"":"downstairs","":"downstairs","session_id":"0d27449f-288f-4651-9ec2-be17ddc0a2d0"} {"msg":"[2]R generation[0..12]: [158, 154, 153, 153, 160, 161, 158, 159, 159, 160, 156, 159]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:19.269360881Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1787,"":"downstairs","":"downstairs","session_id":"0d27449f-288f-4651-9ec2-be17ddc0a2d0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:19.269372061Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1787,"":"downstairs","":"downstairs","session_id":"0d27449f-288f-4651-9ec2-be17ddc0a2d0"} {"msg":"Max found gen is 162","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:19.269381391Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1787,"":"downstairs","":"downstairs","session_id":"0d27449f-288f-4651-9ec2-be17ddc0a2d0"} {"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:19.269392491Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1787,"":"downstairs","":"downstairs","session_id":"0d27449f-288f-4651-9ec2-be17ddc0a2d0"} {"msg":"Next flush: 272","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:19.269401751Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1787,"":"downstairs","":"downstairs","session_id":"0d27449f-288f-4651-9ec2-be17ddc0a2d0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:19.269411181Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1787,"":"downstairs","":"downstairs","session_id":"0d27449f-288f-4651-9ec2-be17ddc0a2d0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:19.269420851Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1787,"session_id":"0d27449f-288f-4651-9ec2-be17ddc0a2d0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:19.269429741Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1787,"client":"0","":"downstairs","session_id":"0d27449f-288f-4651-9ec2-be17ddc0a2d0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:19.269439591Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1787,"client":"1","":"downstairs","session_id":"0d27449f-288f-4651-9ec2-be17ddc0a2d0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:19.269449261Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1787,"client":"2","":"downstairs","session_id":"0d27449f-288f-4651-9ec2-be17ddc0a2d0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:19.269459051Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1787,"session_id":"0d27449f-288f-4651-9ec2-be17ddc0a2d0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:19.269468271Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1787,"session_id":"0d27449f-288f-4651-9ec2-be17ddc0a2d0"} {"msg":"e2ebc6a0-b9b1-4edd-8b35-3de4ad84dec4 is now active with session: 0d27449f-288f-4651-9ec2-be17ddc0a2d0","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:19.269478391Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1787,"session_id":"0d27449f-288f-4651-9ec2-be17ddc0a2d0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:19.269487172Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1787,"session_id":"0d27449f-288f-4651-9ec2-be17ddc0a2d0"} {"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:19.269496242Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1787} {"msg":"Activated sub_volume e2ebc6a0-b9b1-4edd-8b35-3de4ad84dec4","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:19.269504522Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1787} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:19.269512652Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1787,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:19.269846342Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1787,"task":"crutest"} 01/30 Write block 246 len 28672 data: 13 13 13 13 12 15 11 02/30 Write block 425 len 28672 data: 9 9 9 9 9 7 7 03/30 Write block 385 len 4096 data: 14 04/30 Read block 2 len 32768 05/30 Write block 214 len 12288 data: 11 13 14 06/30 Read block 67 len 20480 07/30 Write block 417 len 8192 data: 14 11 08/30 Write block 358 len 40960 data: 11 12 17 13 14 16 18 17 17 13 09/30 Write block 556 len 40960 data: 10 13 9 10 10 11 10 11 9 12 10/30 Flush 11/30 Write block 233 len 8192 data: 10 11 12/30 Read block 318 len 28672 13/30 Read block 187 len 12288 14/30 Read block 127 len 16384 15/30 Write block 120 len 12288 data: 10 10 11 16/30 Read block 108 len 8192 17/30 Read block 492 len 24576 18/30 Write block 109 len 20480 data: 11 11 12 12 14 19/30 Read block 320 len 12288 20/30 Flush 21/30 Write block 412 len 36864 data: 13 13 12 12 13 15 12 10 11 22/30 Write block 405 len 36864 data: 9 7 10 11 12 13 10 14 14 23/30 Read block 177 len 8192 24/30 Write block 42 len 8192 data: 9 9 25/30 Read block 418 len 8192 26/30 Write block 541 len 20480 data: 11 11 15 18 14 27/30 Write block 231 len 32768 data: 6 7 11 12 10 12 16 15 28/30 Read block 481 len 20480 29/30 Write block 203 len 40960 data: 13 12 10 9 9 12 13 13 14 13 30/30 Read block 499 len 4096 ---------------------------------------------------------------- Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Write 7 Done Done Done false 1002 Acked Write 1 Done Done Sent false 1003 Acked Read 8 Done Done Sent false 1004 Acked Write 3 Done Done Sent false 1005 Acked Read 5 Done Done Sent false 1006 Acked Write 2 Done Done Sent false 1007 Acked Write 10 Done Done Sent false 1008 Acked Write 10 Done Done Sent false 1009 Acked Flush 0 Done Done Sent false 1010 Acked Write 2 Done Done Sent false 1011 Acked Read 7 Done Done Sent false 1012 Acked Read 3 Done Done Sent false 1013 Acked Read 4 Done Done Sent false 1014 Acked Write 3 Done Done Sent false 1015 Acked Read 2 Done Done Sent false 1016 Acked Read 6 Done Done Sent false 1017 Acked Write 5 Done Done Sent false 1018 Acked Read 3 Done Done Sent false 1019 Acked Flush 0 Done Done Sent false 1020 Acked Write 9 Done Done Sent false 1021 Acked Write 9 Done Done Sent false 1022 Acked Read 2 Done Done Sent false 1023 Acked Write 2 Done Done Sent false 1024 Acked Read 2 Done Done Sent false 1025 Acked Write 5 Done Done Sent false 1026 Acked Write 8 Done Done Sent false 1027 Acked Read 5 Done Done Sent false 1028 Acked Write 10 Done Sent Sent false 1029 Acked Read 1 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 28 30 Done 30 28 2 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:25:19.278619085Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1787} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:19.278822686Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1787,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 265 265 265 F F F 1 020-039 154 154 154 256 256 256 F F F 2 040-059 153 153 153 254 254 254 T T F <--- 3 060-079 153 153 153 254 254 254 F F F 4 080-099 160 160 160 267 267 267 F F F 5 100-119 162 162 161 273 273 271 F F F <--- 6 120-139 162 162 158 273 273 264 F F F <--- 7 140-159 159 159 159 266 266 266 F F F 8 160-179 159 159 159 266 266 266 F F F 9 180-199 160 160 160 267 267 267 F F F 10 200-219 162 162 156 272 272 259 T T F <--- 11 220-239 162 162 159 273 273 266 T T F <--- 12 240-259 162 162 160 272 272 267 F F T <--- 13 260-279 160 160 160 270 270 270 F F F 14 280-299 156 156 156 259 259 259 F F F 15 300-319 158 158 158 265 265 265 F F F 16 320-339 158 158 158 265 265 265 F F F 17 340-359 162 162 158 272 272 264 F F F <--- 18 360-379 162 162 148 272 272 248 F F F <--- 19 380-399 162 162 161 272 272 271 F F F <--- 20 400-419 162 162 158 272 272 265 T T F <--- 21 420-439 162 162 161 272 272 271 T T T <--- 22 440-459 158 158 158 264 264 264 F F F 23 460-479 156 156 156 261 261 261 F F F 24 480-499 159 159 159 266 266 266 F F F 25 500-519 159 159 159 266 266 266 F F F 26 520-539 157 157 157 262 262 262 F F F 27 540-559 162 162 161 272 272 271 T T F <--- 28 560-579 162 162 160 272 272 267 F F F <--- 29 580-599 160 160 160 268 268 268 F F F Max gen: 162, Max flush: 273 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 81 Verifying data now No change after verify Loop: 81 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 265 265 265 F F F 1 020-039 154 154 154 256 256 256 F F F 2 040-059 163 163 163 274 274 274 F F F 3 060-079 153 153 153 254 254 254 F F F 4 080-099 160 160 160 267 267 267 F F F 5 100-119 162 162 162 273 273 273 F F F 6 120-139 162 162 162 273 273 273 F F F 7 140-159 159 159 159 266 266 266 F F F 8 160-179 159 159 159 266 266 266 F F F 9 180-199 160 160 160 267 267 267 F F F 10 200-219 163 163 163 274 274 274 F F F 11 220-239 163 163 163 274 274 274 F F F 12 240-259 162 162 162 272 272 272 F F F 13 260-279 160 160 160 270 270 270 F F F 14 280-299 156 156 156 259 259 259 F F F 15 300-319 158 158 158 265 265 265 F F F 16 320-339 158 158 158 265 265 265 F F F 17 340-359 162 162 162 272 272 272 F F F 18 360-379 162 162 162 272 272 272 F F F 19 380-399 162 162 162 272 272 272 F F F 20 400-419 163 163 163 274 274 274 F F F 21 420-439 163 163 163 274 274 274 F F F 22 440-459 158 158 158 264 264 264 F F F 23 460-479 156 156 156 261 261 261 F F F 24 480-499 159 159 159 266 266 266 F F F 25 500-519 159 159 159 266 266 266 F F F 26 520-539 157 157 157 262 262 262 F F F 27 540-559 163 163 163 274 274 274 F F F 28 560-579 162 162 162 272 272 272 F F F 29 580-599 160 160 160 268 268 268 F F F Max gen: 163, Max flush: 274 Begin loop 82 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:23.531846064Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1795,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:25:23.539105194Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1795,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:23.539161715Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1795,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:23.546139455Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1795,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:23.546408966Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1795} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:23.546436126Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1795} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:23.546444596Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1795} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:23.546456326Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1795} {"msg":"Crucible a33bf71d-f885-4ea1-9235-3eea5ba04d73 has session id: 2fdc23ae-fe17-4579-81c4-160883aae0d7","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:23.546797476Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1795,"session_id":"2fdc23ae-fe17-4579-81c4-160883aae0d7"} {"msg":"Upstairs opts: Upstairs UUID: a33bf71d-f885-4ea1-9235-3eea5ba04d73, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:23.546827216Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1795,"session_id":"2fdc23ae-fe17-4579-81c4-160883aae0d7"} {"msg":"Crucible stats registered with UUID: a33bf71d-f885-4ea1-9235-3eea5ba04d73","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:23.546838746Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1795,"session_id":"2fdc23ae-fe17-4579-81c4-160883aae0d7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:25:23.546848146Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1795,"session_id":"2fdc23ae-fe17-4579-81c4-160883aae0d7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:23.546968996Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1795,"":"io task","client":"0","":"downstairs","session_id":"2fdc23ae-fe17-4579-81c4-160883aae0d7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:23.546986916Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1795,"":"io task","client":"2","":"downstairs","session_id":"2fdc23ae-fe17-4579-81c4-160883aae0d7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:23.546997166Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1795,"":"io task","client":"1","":"downstairs","session_id":"2fdc23ae-fe17-4579-81c4-160883aae0d7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:23.547178957Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1795,"task":"crutest"} {"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:23.547198127Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1795} {"msg":"a33bf71d-f885-4ea1-9235-3eea5ba04d73 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:23.547222757Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1795,"session_id":"2fdc23ae-fe17-4579-81c4-160883aae0d7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:23.547232387Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1795,"client":"0","":"downstairs","session_id":"2fdc23ae-fe17-4579-81c4-160883aae0d7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:23.547242167Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1795,"client":"1","":"downstairs","session_id":"2fdc23ae-fe17-4579-81c4-160883aae0d7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:23.547252107Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1795,"client":"2","":"downstairs","session_id":"2fdc23ae-fe17-4579-81c4-160883aae0d7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:23.547261817Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1795,"":"io task","client":"2","":"downstairs","session_id":"2fdc23ae-fe17-4579-81c4-160883aae0d7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:23.547285897Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1795,"":"io task","client":"0","":"downstairs","session_id":"2fdc23ae-fe17-4579-81c4-160883aae0d7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:23.547296457Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1795,"":"io task","client":"1","":"downstairs","session_id":"2fdc23ae-fe17-4579-81c4-160883aae0d7"} {"msg":"ds_connection connected from Ok(127.0.0.1:35355)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:23.547329577Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1795,"":"io task","client":"1","":"downstairs","session_id":"2fdc23ae-fe17-4579-81c4-160883aae0d7"} {"msg":"ds_connection connected from Ok(127.0.0.1:47179)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:23.547340637Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1795,"":"io task","client":"2","":"downstairs","session_id":"2fdc23ae-fe17-4579-81c4-160883aae0d7"} {"msg":"ds_connection connected from Ok(127.0.0.1:37591)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:23.547445767Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1795,"":"io task","client":"0","":"downstairs","session_id":"2fdc23ae-fe17-4579-81c4-160883aae0d7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:23.548040088Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1795,"client":"1","":"downstairs","session_id":"2fdc23ae-fe17-4579-81c4-160883aae0d7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:23.548076198Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1795,"client":"0","":"downstairs","session_id":"2fdc23ae-fe17-4579-81c4-160883aae0d7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:23.548094298Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1795,"client":"2","":"downstairs","session_id":"2fdc23ae-fe17-4579-81c4-160883aae0d7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:23.548340998Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1795,"session_id":"2fdc23ae-fe17-4579-81c4-160883aae0d7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:23.548373218Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1795,"session_id":"2fdc23ae-fe17-4579-81c4-160883aae0d7"} {"msg":"[0]R flush_numbers[0..12]: [265, 256, 274, 254, 267, 273, 273, 266, 266, 267, 274, 274]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:23.548416949Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1795,"":"downstairs","":"downstairs","session_id":"2fdc23ae-fe17-4579-81c4-160883aae0d7"} {"msg":"[0]R generation[0..12]: [158, 154, 163, 153, 160, 162, 162, 159, 159, 160, 163, 163]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:23.548428279Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1795,"":"downstairs","":"downstairs","session_id":"2fdc23ae-fe17-4579-81c4-160883aae0d7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:23.548439489Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1795,"":"downstairs","":"downstairs","session_id":"2fdc23ae-fe17-4579-81c4-160883aae0d7"} {"msg":"[1]R flush_numbers[0..12]: [265, 256, 274, 254, 267, 273, 273, 266, 266, 267, 274, 274]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:23.548450229Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1795,"":"downstairs","":"downstairs","session_id":"2fdc23ae-fe17-4579-81c4-160883aae0d7"} {"msg":"[1]R generation[0..12]: [158, 154, 163, 153, 160, 162, 162, 159, 159, 160, 163, 163]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:23.548460979Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1795,"":"downstairs","":"downstairs","session_id":"2fdc23ae-fe17-4579-81c4-160883aae0d7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:23.548471949Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1795,"":"downstairs","":"downstairs","session_id":"2fdc23ae-fe17-4579-81c4-160883aae0d7"} {"msg":"[2]R flush_numbers[0..12]: [265, 256, 274, 254, 267, 273, 273, 266, 266, 267, 274, 274]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:23.548482719Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1795,"":"downstairs","":"downstairs","session_id":"2fdc23ae-fe17-4579-81c4-160883aae0d7"} {"msg":"[2]R generation[0..12]: [158, 154, 163, 153, 160, 162, 162, 159, 159, 160, 163, 163]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:23.548493379Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1795,"":"downstairs","":"downstairs","session_id":"2fdc23ae-fe17-4579-81c4-160883aae0d7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:23.548504269Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1795,"":"downstairs","":"downstairs","session_id":"2fdc23ae-fe17-4579-81c4-160883aae0d7"} {"msg":"Max found gen is 164","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:23.548513709Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1795,"":"downstairs","":"downstairs","session_id":"2fdc23ae-fe17-4579-81c4-160883aae0d7"} {"msg":"Generation requested: 164 >= found:164","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:23.548523599Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1795,"":"downstairs","":"downstairs","session_id":"2fdc23ae-fe17-4579-81c4-160883aae0d7"} {"msg":"Next flush: 275","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:23.548533059Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1795,"":"downstairs","":"downstairs","session_id":"2fdc23ae-fe17-4579-81c4-160883aae0d7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:23.548542529Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1795,"":"downstairs","":"downstairs","session_id":"2fdc23ae-fe17-4579-81c4-160883aae0d7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:23.548553779Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1795,"session_id":"2fdc23ae-fe17-4579-81c4-160883aae0d7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:23.548562659Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1795,"client":"0","":"downstairs","session_id":"2fdc23ae-fe17-4579-81c4-160883aae0d7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:23.548572699Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1795,"client":"1","":"downstairs","session_id":"2fdc23ae-fe17-4579-81c4-160883aae0d7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:23.548582609Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1795,"client":"2","":"downstairs","session_id":"2fdc23ae-fe17-4579-81c4-160883aae0d7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:23.548592719Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1795,"session_id":"2fdc23ae-fe17-4579-81c4-160883aae0d7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:23.548601939Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1795,"session_id":"2fdc23ae-fe17-4579-81c4-160883aae0d7"} {"msg":"a33bf71d-f885-4ea1-9235-3eea5ba04d73 is now active with session: 2fdc23ae-fe17-4579-81c4-160883aae0d7","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:23.548612109Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1795,"session_id":"2fdc23ae-fe17-4579-81c4-160883aae0d7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:23.548620899Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1795,"session_id":"2fdc23ae-fe17-4579-81c4-160883aae0d7"} {"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:23.548630179Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1795} {"msg":"Activated sub_volume a33bf71d-f885-4ea1-9235-3eea5ba04d73","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:23.548638299Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1795} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:23.548646559Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1795,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:23.548971589Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1795,"task":"crutest"} 01/30 Write block 410 len 16384 data: 14 11 15 15 02/30 Write block 296 len 28672 data: 16 16 18 17 15 16 16 03/30 Write block 226 len 24576 data: 10 10 11 8 8 7 04/30 Read block 388 len 4096 05/30 Read block 546 len 32768 06/30 Write block 488 len 28672 data: 18 18 17 11 13 15 14 07/30 Write block 426 len 24576 data: 10 10 10 10 8 8 08/30 Flush 09/30 Write block 165 len 12288 data: 7 7 11 10/30 Write block 171 len 12288 data: 14 15 15 11/30 Write block 515 len 8192 data: 10 9 12/30 Read block 525 len 8192 13/30 Read block 236 len 4096 14/30 Write block 377 len 40960 data: 15 14 16 15 16 12 13 13 15 14 15/30 Write block 305 len 16384 data: 13 12 11 8 16/30 Read block 300 len 4096 17/30 Read block 464 len 24576 18/30 Read block 305 len 4096 19/30 Flush 20/30 Write block 483 len 28672 data: 9 11 13 15 18 19 19 21/30 Read block 229 len 16384 22/30 Write block 583 len 36864 data: 11 13 11 12 14 13 10 8 8 23/30 Read block 214 len 24576 24/30 Read block 519 len 32768 25/30 Flush 26/30 Write block 413 len 32768 data: 16 13 13 14 16 13 11 12 27/30 Read block 20 len 24576 28/30 Write block 476 len 12288 data: 11 12 12 29/30 Write block 277 len 8192 data: 17 12 30/30 Read block 446 len 28672 ---------------------------------------------------------------- Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Write 7 Done Done Done false 1002 Acked Write 6 Sent Done Done false 1003 Acked Read 1 Sent Done Done false 1004 Acked Read 8 Sent Done Done false 1005 Acked Write 7 Sent Done Done false 1006 Acked Write 6 Sent Done Done false 1007 Acked Flush 0 Sent Done Done false 1008 Acked Write 3 Sent Done Done false 1009 Acked Write 3 Sent Done Done false 1010 Acked Write 2 Sent Done Done false 1011 Acked Read 2 Sent Done Done false 1012 Acked Read 1 Sent Done Done false 1013 Acked Write 10 Sent Done Done false 1014 Acked Write 4 Sent Done Done false 1015 Acked Read 1 Sent Done Done false 1016 Acked Read 6 Sent Done Done false 1017 Acked Read 1 Sent Done Done false 1018 Acked Flush 0 Sent Done Done false 1019 Acked Write 7 Sent Done Done false 1020 Acked Read 4 Sent Done Done false 1021 Acked Write 9 Sent Done Done false 1022 Acked Read 6 Sent Done Done false 1023 Acked Read 8 Sent Done Done false 1024 Acked Flush 0 Sent Done Done false 1025 Acked Write 8 Sent Done Done false 1026 Acked Read 6 Sent Done Done false 1027 Acked Write 3 Sent Done Done false 1028 Acked Write 2 Sent Done Done false 1029 Acked Read 7 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 0 1 29 Done 2 30 29 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:25:23.557805123Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1795} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:23.558010803Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1795,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 265 265 265 F F F 1 020-039 154 154 154 256 256 256 F F F 2 040-059 163 163 163 274 274 274 F F F 3 060-079 153 153 153 254 254 254 F F F 4 080-099 160 160 160 267 267 267 F F F 5 100-119 162 162 162 273 273 273 F F F 6 120-139 162 162 162 273 273 273 F F F 7 140-159 159 159 159 266 266 266 F F F 8 160-179 159 164 164 266 276 276 F F F <--- 9 180-199 160 160 160 267 267 267 F F F 10 200-219 163 163 163 274 274 274 F F F 11 220-239 163 164 164 274 275 275 F F F <--- 12 240-259 162 162 162 272 272 272 F F F 13 260-279 160 160 160 270 270 270 F T T <--- 14 280-299 156 164 164 259 275 275 T F F <--- 15 300-319 158 164 164 265 276 276 T F F <--- 16 320-339 158 158 158 265 265 265 F F F 17 340-359 162 162 162 272 272 272 F F F 18 360-379 162 164 164 272 276 276 F F F <--- 19 380-399 162 164 164 272 276 276 F F F <--- 20 400-419 163 164 164 274 275 275 T T T <--- 21 420-439 163 164 164 274 275 275 F T T <--- 22 440-459 158 158 158 264 264 264 F F F 23 460-479 156 156 156 261 261 261 F T T <--- 24 480-499 159 164 164 266 277 277 F F F <--- 25 500-519 159 164 164 266 276 276 F F F <--- 26 520-539 157 157 157 262 262 262 F F F 27 540-559 163 163 163 274 274 274 F F F 28 560-579 162 162 162 272 272 272 F F F 29 580-599 160 164 164 268 277 277 F F F <--- Max gen: 164, Max flush: 277 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 82 Verifying data now No change after verify Loop: 82 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 265 265 265 F F F 1 020-039 154 154 154 256 256 256 F F F 2 040-059 163 163 163 274 274 274 F F F 3 060-079 153 153 153 254 254 254 F F F 4 080-099 160 160 160 267 267 267 F F F 5 100-119 162 162 162 273 273 273 F F F 6 120-139 162 162 162 273 273 273 F F F 7 140-159 159 159 159 266 266 266 F F F 8 160-179 164 164 164 276 276 276 F F F 9 180-199 160 160 160 267 267 267 F F F 10 200-219 163 163 163 274 274 274 F F F 11 220-239 164 164 164 275 275 275 F F F 12 240-259 162 162 162 272 272 272 F F F 13 260-279 165 165 165 278 278 278 F F F 14 280-299 164 164 164 275 275 275 F F F 15 300-319 164 164 164 276 276 276 F F F 16 320-339 158 158 158 265 265 265 F F F 17 340-359 162 162 162 272 272 272 F F F 18 360-379 164 164 164 276 276 276 F F F 19 380-399 164 164 164 276 276 276 F F F 20 400-419 165 165 165 278 278 278 F F F 21 420-439 165 165 165 278 278 278 F F F 22 440-459 158 158 158 264 264 264 F F F 23 460-479 165 165 165 278 278 278 F F F 24 480-499 164 164 164 277 277 277 F F F 25 500-519 164 164 164 276 276 276 F F F 26 520-539 157 157 157 262 262 262 F F F 27 540-559 163 163 163 274 274 274 F F F 28 560-579 162 162 162 272 272 272 F F F 29 580-599 164 164 164 277 277 277 F F F Max gen: 165, Max flush: 278 Begin loop 83 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:27.815445108Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1803,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:25:27.822623548Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1803,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:27.822678468Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1803,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:27.829579978Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1803,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:27.829867189Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1803} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:27.829894279Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1803} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:27.829905449Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1803} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:27.829917269Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1803} {"msg":"Crucible a9131b54-1543-4e5f-b025-aebf13bc7616 has session id: 625d6c12-e375-437d-ac92-80484b2397cd","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:27.83027843Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1803,"session_id":"625d6c12-e375-437d-ac92-80484b2397cd"} {"msg":"Upstairs opts: Upstairs UUID: a9131b54-1543-4e5f-b025-aebf13bc7616, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:27.83030793Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1803,"session_id":"625d6c12-e375-437d-ac92-80484b2397cd"} {"msg":"Crucible stats registered with UUID: a9131b54-1543-4e5f-b025-aebf13bc7616","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:27.83031858Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1803,"session_id":"625d6c12-e375-437d-ac92-80484b2397cd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:25:27.83032788Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1803,"session_id":"625d6c12-e375-437d-ac92-80484b2397cd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:27.83046362Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1803,"":"io task","client":"1","":"downstairs","session_id":"625d6c12-e375-437d-ac92-80484b2397cd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:27.83049613Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1803,"":"io task","client":"0","":"downstairs","session_id":"625d6c12-e375-437d-ac92-80484b2397cd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:27.83050692Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1803,"":"io task","client":"2","":"downstairs","session_id":"625d6c12-e375-437d-ac92-80484b2397cd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:27.8306342Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1803,"task":"crutest"} {"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:27.83064515Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1803} {"msg":"a9131b54-1543-4e5f-b025-aebf13bc7616 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:27.83067663Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1803,"session_id":"625d6c12-e375-437d-ac92-80484b2397cd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:27.83068631Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1803,"client":"0","":"downstairs","session_id":"625d6c12-e375-437d-ac92-80484b2397cd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:27.83069791Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1803,"client":"1","":"downstairs","session_id":"625d6c12-e375-437d-ac92-80484b2397cd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:27.83070788Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1803,"client":"2","":"downstairs","session_id":"625d6c12-e375-437d-ac92-80484b2397cd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:27.83071747Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1803,"":"io task","client":"2","":"downstairs","session_id":"625d6c12-e375-437d-ac92-80484b2397cd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:27.83072765Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1803,"":"io task","client":"0","":"downstairs","session_id":"625d6c12-e375-437d-ac92-80484b2397cd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:27.83073768Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1803,"":"io task","client":"1","":"downstairs","session_id":"625d6c12-e375-437d-ac92-80484b2397cd"} {"msg":"ds_connection connected from Ok(127.0.0.1:35279)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:27.83078628Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1803,"":"io task","client":"2","":"downstairs","session_id":"625d6c12-e375-437d-ac92-80484b2397cd"} {"msg":"ds_connection connected from Ok(127.0.0.1:50457)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:27.83079737Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1803,"":"io task","client":"1","":"downstairs","session_id":"625d6c12-e375-437d-ac92-80484b2397cd"} {"msg":"ds_connection connected from Ok(127.0.0.1:33971)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:27.83081024Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1803,"":"io task","client":"0","":"downstairs","session_id":"625d6c12-e375-437d-ac92-80484b2397cd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:27.831540961Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1803,"client":"1","":"downstairs","session_id":"625d6c12-e375-437d-ac92-80484b2397cd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:27.831582691Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1803,"client":"2","":"downstairs","session_id":"625d6c12-e375-437d-ac92-80484b2397cd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:27.831604572Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1803,"client":"0","":"downstairs","session_id":"625d6c12-e375-437d-ac92-80484b2397cd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:27.831887842Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1803,"session_id":"625d6c12-e375-437d-ac92-80484b2397cd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:27.831898142Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1803,"session_id":"625d6c12-e375-437d-ac92-80484b2397cd"} {"msg":"[0]R flush_numbers[0..12]: [265, 256, 274, 254, 267, 273, 273, 266, 276, 267, 274, 275]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:27.831935912Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1803,"":"downstairs","":"downstairs","session_id":"625d6c12-e375-437d-ac92-80484b2397cd"} {"msg":"[0]R generation[0..12]: [158, 154, 163, 153, 160, 162, 162, 159, 164, 160, 163, 164]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:27.831947352Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1803,"":"downstairs","":"downstairs","session_id":"625d6c12-e375-437d-ac92-80484b2397cd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:27.831961122Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1803,"":"downstairs","":"downstairs","session_id":"625d6c12-e375-437d-ac92-80484b2397cd"} {"msg":"[1]R flush_numbers[0..12]: [265, 256, 274, 254, 267, 273, 273, 266, 276, 267, 274, 275]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:27.831971942Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1803,"":"downstairs","":"downstairs","session_id":"625d6c12-e375-437d-ac92-80484b2397cd"} {"msg":"[1]R generation[0..12]: [158, 154, 163, 153, 160, 162, 162, 159, 164, 160, 163, 164]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:27.831982702Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1803,"":"downstairs","":"downstairs","session_id":"625d6c12-e375-437d-ac92-80484b2397cd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:27.831993742Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1803,"":"downstairs","":"downstairs","session_id":"625d6c12-e375-437d-ac92-80484b2397cd"} {"msg":"[2]R flush_numbers[0..12]: [265, 256, 274, 254, 267, 273, 273, 266, 276, 267, 274, 275]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:27.832004592Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1803,"":"downstairs","":"downstairs","session_id":"625d6c12-e375-437d-ac92-80484b2397cd"} {"msg":"[2]R generation[0..12]: [158, 154, 163, 153, 160, 162, 162, 159, 164, 160, 163, 164]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:27.832015452Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1803,"":"downstairs","":"downstairs","session_id":"625d6c12-e375-437d-ac92-80484b2397cd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:27.832026392Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1803,"":"downstairs","":"downstairs","session_id":"625d6c12-e375-437d-ac92-80484b2397cd"} {"msg":"Max found gen is 166","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:27.832035932Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1803,"":"downstairs","":"downstairs","session_id":"625d6c12-e375-437d-ac92-80484b2397cd"} {"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:27.832045622Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1803,"":"downstairs","":"downstairs","session_id":"625d6c12-e375-437d-ac92-80484b2397cd"} {"msg":"Next flush: 279","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:27.832054992Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1803,"":"downstairs","":"downstairs","session_id":"625d6c12-e375-437d-ac92-80484b2397cd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:27.832064412Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1803,"":"downstairs","":"downstairs","session_id":"625d6c12-e375-437d-ac92-80484b2397cd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:27.832074342Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1803,"session_id":"625d6c12-e375-437d-ac92-80484b2397cd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:27.832083292Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1803,"client":"0","":"downstairs","session_id":"625d6c12-e375-437d-ac92-80484b2397cd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:27.832093152Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1803,"client":"1","":"downstairs","session_id":"625d6c12-e375-437d-ac92-80484b2397cd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:27.832102952Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1803,"client":"2","":"downstairs","session_id":"625d6c12-e375-437d-ac92-80484b2397cd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:27.832114302Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1803,"session_id":"625d6c12-e375-437d-ac92-80484b2397cd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:27.832123532Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1803,"session_id":"625d6c12-e375-437d-ac92-80484b2397cd"} {"msg":"a9131b54-1543-4e5f-b025-aebf13bc7616 is now active with session: 625d6c12-e375-437d-ac92-80484b2397cd","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:27.832133652Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1803,"session_id":"625d6c12-e375-437d-ac92-80484b2397cd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:27.832142392Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1803,"session_id":"625d6c12-e375-437d-ac92-80484b2397cd"} {"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:27.832151592Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1803} {"msg":"Activated sub_volume a9131b54-1543-4e5f-b025-aebf13bc7616","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:27.832159582Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1803} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:27.832167912Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1803,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:27.832480553Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1803,"task":"crutest"} 01/30 Write block 62 len 28672 data: 15 15 13 12 13 12 9 02/30 Read block 280 len 24576 03/30 Read block 255 len 20480 04/30 Write block 110 len 32768 data: 12 13 13 15 14 14 14 11 05/30 Read block 534 len 12288 06/30 Flush 07/30 Write block 309 len 20480 data: 8 9 7 7 8 08/30 Write block 78 len 28672 data: 15 16 16 12 12 16 15 09/30 Read block 393 len 28672 10/30 Write block 191 len 20480 data: 15 16 16 12 13 11/30 Read block 368 len 40960 12/30 Write block 121 len 28672 data: 11 12 11 10 11 9 7 13/30 Write block 49 len 20480 data: 8 12 13 15 14 14/30 Read block 98 len 32768 15/30 Flush 16/30 Write block 211 len 28672 data: 15 14 12 12 14 15 14 17/30 Write block 125 len 28672 data: 12 10 8 9 8 6 8 18/30 Read block 574 len 12288 19/30 Write block 260 len 32768 data: 12 14 13 15 14 15 14 16 20/30 Read block 265 len 32768 21/30 Write block 582 len 32768 data: 10 12 14 12 13 15 14 11 22/30 Write block 580 len 40960 data: 12 10 11 13 15 13 14 16 15 12 23/30 Write block 219 len 28672 data: 14 10 9 9 11 8 10 24/30 Flush 25/30 Flush 26/30 Write block 411 len 28672 data: 12 16 17 14 14 15 17 27/30 Write block 595 len 8192 data: 9 7 28/30 Write block 516 len 28672 data: 10 10 10 13 14 14 12 29/30 Read block 396 len 4096 30/30 Write block 137 len 28672 data: 11 14 17 15 13 11 9 ---------------------------------------------------------------- Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Read 6 Done Done Done false 1002 Acked Read 5 Done Done Done false 1003 Acked Write 8 Done Done Done false 1004 Acked Read 3 Done Done Done false 1005 Acked Flush 0 Sent Done Done false 1006 Acked Write 5 Sent Done Done false 1007 Acked Write 7 Sent Done Done false 1008 Acked Read 7 Sent Done Done false 1009 Acked Write 5 Sent Done Done false 1010 Acked Read 10 Sent Done Done false 1011 Acked Write 7 Sent Done Done false 1012 Acked Write 5 Sent Done Done false 1013 Acked Read 8 Sent Done Done false 1014 Acked Flush 0 Sent Done Done false 1015 Acked Write 7 Sent Done Done false 1016 Acked Write 7 Sent Done Done false 1017 Acked Read 3 Sent Done Done false 1018 Acked Write 8 Sent Done Done false 1019 Acked Read 8 Sent Done Done false 1020 Acked Write 8 Sent Done Done false 1021 Acked Write 10 Sent Done Done false 1022 Acked Write 7 Sent Done Done false 1023 Acked Flush 0 Sent Done Done false 1024 Acked Flush 0 Sent Done Done false 1025 Acked Write 7 Sent Done Done false 1026 Acked Write 2 Sent Done Done false 1027 Acked Write 7 Sent Done Done false 1028 Acked Read 1 Sent Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 25 1 1 27 Done 5 29 29 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:25:27.841886767Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1803} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:27.842081147Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1803,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 265 265 265 F F F 1 020-039 154 154 154 256 256 256 F F F 2 040-059 163 166 166 274 280 280 F F F <--- 3 060-079 153 166 166 254 280 280 T F F <--- 4 080-099 160 166 166 267 280 280 F F F <--- 5 100-119 162 166 166 273 279 279 T F F <--- 6 120-139 162 166 166 273 281 281 F T T <--- 7 140-159 159 159 159 266 266 266 F T T <--- 8 160-179 164 164 164 276 276 276 F F F 9 180-199 160 166 166 267 280 280 F F F <--- 10 200-219 163 166 166 274 281 281 F F F <--- 11 220-239 164 166 166 275 281 281 F F F <--- 12 240-259 162 162 162 272 272 272 F F F 13 260-279 165 166 166 278 281 281 F F F <--- 14 280-299 164 164 164 275 275 275 F F F 15 300-319 164 166 166 276 280 280 F F F <--- 16 320-339 158 158 158 265 265 265 F F F 17 340-359 162 162 162 272 272 272 F F F 18 360-379 164 164 164 276 276 276 F F F 19 380-399 164 164 164 276 276 276 F F F 20 400-419 165 165 165 278 278 278 F T T <--- 21 420-439 165 165 165 278 278 278 F F F 22 440-459 158 158 158 264 264 264 F F F 23 460-479 165 165 165 278 278 278 F F F 24 480-499 164 164 164 277 277 277 F F F 25 500-519 164 164 164 276 276 276 F T T <--- 26 520-539 157 157 157 262 262 262 F T T <--- 27 540-559 163 163 163 274 274 274 F F F 28 560-579 162 162 162 272 272 272 F F F 29 580-599 164 166 166 277 281 281 F T T <--- Max gen: 166, Max flush: 281 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 83 Verifying data now No change after verify Loop: 83 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 265 265 265 F F F 1 020-039 154 154 154 256 256 256 F F F 2 040-059 166 166 166 280 280 280 F F F 3 060-079 166 166 166 280 280 280 F F F 4 080-099 166 166 166 280 280 280 F F F 5 100-119 166 166 166 279 279 279 F F F 6 120-139 167 167 167 282 282 282 F F F 7 140-159 167 167 167 282 282 282 F F F 8 160-179 164 164 164 276 276 276 F F F 9 180-199 166 166 166 280 280 280 F F F 10 200-219 166 166 166 281 281 281 F F F 11 220-239 166 166 166 281 281 281 F F F 12 240-259 162 162 162 272 272 272 F F F 13 260-279 166 166 166 281 281 281 F F F 14 280-299 164 164 164 275 275 275 F F F 15 300-319 166 166 166 280 280 280 F F F 16 320-339 158 158 158 265 265 265 F F F 17 340-359 162 162 162 272 272 272 F F F 18 360-379 164 164 164 276 276 276 F F F 19 380-399 164 164 164 276 276 276 F F F 20 400-419 167 167 167 282 282 282 F F F 21 420-439 165 165 165 278 278 278 F F F 22 440-459 158 158 158 264 264 264 F F F 23 460-479 165 165 165 278 278 278 F F F 24 480-499 164 164 164 277 277 277 F F F 25 500-519 167 167 167 282 282 282 F F F 26 520-539 167 167 167 282 282 282 F F F 27 540-559 163 163 163 274 274 274 F F F 28 560-579 162 162 162 272 272 272 F F F 29 580-599 167 167 167 282 282 282 F F F Max gen: 167, Max flush: 282 Begin loop 84 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:32.107213474Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1811,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:32.115591016Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1811,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:32.115892437Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1811} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:32.115917937Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1811} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:32.115926767Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1811} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:32.115938627Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1811} {"msg":"Crucible 45455305-edca-4cb3-b9e2-8d7a710c20a4 has session id: d9feb4b2-c177-4e2d-a98c-2345292ce082","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:32.116300377Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1811,"session_id":"d9feb4b2-c177-4e2d-a98c-2345292ce082"} {"msg":"Upstairs opts: Upstairs UUID: 45455305-edca-4cb3-b9e2-8d7a710c20a4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:32.116334137Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1811,"session_id":"d9feb4b2-c177-4e2d-a98c-2345292ce082"} {"msg":"Crucible stats registered with UUID: 45455305-edca-4cb3-b9e2-8d7a710c20a4","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:32.116345227Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1811,"session_id":"d9feb4b2-c177-4e2d-a98c-2345292ce082"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:25:32.116354487Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1811,"session_id":"d9feb4b2-c177-4e2d-a98c-2345292ce082"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:32.116494168Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1811,"":"io task","client":"2","":"downstairs","session_id":"d9feb4b2-c177-4e2d-a98c-2345292ce082"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:32.116525418Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1811,"":"io task","client":"0","":"downstairs","session_id":"d9feb4b2-c177-4e2d-a98c-2345292ce082"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:32.116536348Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1811,"":"io task","client":"1","":"downstairs","session_id":"d9feb4b2-c177-4e2d-a98c-2345292ce082"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:32.116661048Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1811,"task":"crutest"} {"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:32.116680058Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1811} {"msg":"45455305-edca-4cb3-b9e2-8d7a710c20a4 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:32.116696028Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1811,"session_id":"d9feb4b2-c177-4e2d-a98c-2345292ce082"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:32.116705048Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1811,"client":"0","":"downstairs","session_id":"d9feb4b2-c177-4e2d-a98c-2345292ce082"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:32.116714828Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1811,"client":"1","":"downstairs","session_id":"d9feb4b2-c177-4e2d-a98c-2345292ce082"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:32.116724498Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1811,"client":"2","":"downstairs","session_id":"d9feb4b2-c177-4e2d-a98c-2345292ce082"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:32.116734098Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1811,"":"io task","client":"2","":"downstairs","session_id":"d9feb4b2-c177-4e2d-a98c-2345292ce082"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:32.116744298Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1811,"":"io task","client":"0","":"downstairs","session_id":"d9feb4b2-c177-4e2d-a98c-2345292ce082"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:32.116754368Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1811,"":"io task","client":"1","":"downstairs","session_id":"d9feb4b2-c177-4e2d-a98c-2345292ce082"} {"msg":"ds_connection connected from Ok(127.0.0.1:48971)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:32.116818098Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1811,"":"io task","client":"1","":"downstairs","session_id":"d9feb4b2-c177-4e2d-a98c-2345292ce082"} {"msg":"ds_connection connected from Ok(127.0.0.1:57015)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:32.116849058Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1811,"":"io task","client":"2","":"downstairs","session_id":"d9feb4b2-c177-4e2d-a98c-2345292ce082"} {"msg":"ds_connection connected from Ok(127.0.0.1:46483)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:32.116883578Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1811,"":"io task","client":"0","":"downstairs","session_id":"d9feb4b2-c177-4e2d-a98c-2345292ce082"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:32.117611459Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1811,"client":"0","":"downstairs","session_id":"d9feb4b2-c177-4e2d-a98c-2345292ce082"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:32.117634599Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1811,"client":"1","":"downstairs","session_id":"d9feb4b2-c177-4e2d-a98c-2345292ce082"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:32.117666319Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1811,"client":"2","":"downstairs","session_id":"d9feb4b2-c177-4e2d-a98c-2345292ce082"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:32.11795954Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1811,"session_id":"d9feb4b2-c177-4e2d-a98c-2345292ce082"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:32.11797193Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1811,"session_id":"d9feb4b2-c177-4e2d-a98c-2345292ce082"} {"msg":"[0]R flush_numbers[0..12]: [265, 256, 280, 280, 280, 279, 282, 282, 276, 280, 281, 281]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:32.11800697Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1811,"":"downstairs","":"downstairs","session_id":"d9feb4b2-c177-4e2d-a98c-2345292ce082"} {"msg":"[0]R generation[0..12]: [158, 154, 166, 166, 166, 166, 167, 167, 164, 166, 166, 166]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:32.11801851Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1811,"":"downstairs","":"downstairs","session_id":"d9feb4b2-c177-4e2d-a98c-2345292ce082"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:32.11802964Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1811,"":"downstairs","":"downstairs","session_id":"d9feb4b2-c177-4e2d-a98c-2345292ce082"} {"msg":"[1]R flush_numbers[0..12]: [265, 256, 280, 280, 280, 279, 282, 282, 276, 280, 281, 281]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:32.1180404Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1811,"":"downstairs","":"downstairs","session_id":"d9feb4b2-c177-4e2d-a98c-2345292ce082"} {"msg":"[1]R generation[0..12]: [158, 154, 166, 166, 166, 166, 167, 167, 164, 166, 166, 166]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:32.11805144Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1811,"":"downstairs","":"downstairs","session_id":"d9feb4b2-c177-4e2d-a98c-2345292ce082"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:32.11806256Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1811,"":"downstairs","":"downstairs","session_id":"d9feb4b2-c177-4e2d-a98c-2345292ce082"} {"msg":"[2]R flush_numbers[0..12]: [265, 256, 280, 280, 280, 279, 282, 282, 276, 280, 281, 281]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:32.11807591Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1811,"":"downstairs","":"downstairs","session_id":"d9feb4b2-c177-4e2d-a98c-2345292ce082"} {"msg":"[2]R generation[0..12]: [158, 154, 166, 166, 166, 166, 167, 167, 164, 166, 166, 166]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:32.11808666Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1811,"":"downstairs","":"downstairs","session_id":"d9feb4b2-c177-4e2d-a98c-2345292ce082"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:32.1180976Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1811,"":"downstairs","":"downstairs","session_id":"d9feb4b2-c177-4e2d-a98c-2345292ce082"} {"msg":"Max found gen is 168","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:32.11810717Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1811,"":"downstairs","":"downstairs","session_id":"d9feb4b2-c177-4e2d-a98c-2345292ce082"} {"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:32.118117Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1811,"":"downstairs","":"downstairs","session_id":"d9feb4b2-c177-4e2d-a98c-2345292ce082"} {"msg":"Next flush: 283","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:32.11812656Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1811,"":"downstairs","":"downstairs","session_id":"d9feb4b2-c177-4e2d-a98c-2345292ce082"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:32.11813592Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1811,"":"downstairs","":"downstairs","session_id":"d9feb4b2-c177-4e2d-a98c-2345292ce082"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:32.11814583Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1811,"session_id":"d9feb4b2-c177-4e2d-a98c-2345292ce082"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:32.11815477Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1811,"client":"0","":"downstairs","session_id":"d9feb4b2-c177-4e2d-a98c-2345292ce082"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:32.11816452Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1811,"client":"1","":"downstairs","session_id":"d9feb4b2-c177-4e2d-a98c-2345292ce082"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:32.11817422Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1811,"client":"2","":"downstairs","session_id":"d9feb4b2-c177-4e2d-a98c-2345292ce082"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:32.11818416Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1811,"session_id":"d9feb4b2-c177-4e2d-a98c-2345292ce082"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:32.11819342Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1811,"session_id":"d9feb4b2-c177-4e2d-a98c-2345292ce082"} {"msg":"45455305-edca-4cb3-b9e2-8d7a710c20a4 is now active with session: d9feb4b2-c177-4e2d-a98c-2345292ce082","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:32.11820356Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1811,"session_id":"d9feb4b2-c177-4e2d-a98c-2345292ce082"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:32.11821236Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1811,"session_id":"d9feb4b2-c177-4e2d-a98c-2345292ce082"} {"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:32.11822193Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1811} {"msg":"Activated sub_volume 45455305-edca-4cb3-b9e2-8d7a710c20a4","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:32.11823158Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1811} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:32.11823971Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1811,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:32.118543251Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1811,"task":"crutest"} 01/30 Write block 156 len 16384 data: 14 14 17 14 02/30 Read block 583 len 20480 03/30 Read block 145 len 16384 04/30 Read block 379 len 28672 05/30 Flush 06/30 Write block 249 len 12288 data: 14 13 16 07/30 Write block 67 len 20480 data: 13 10 12 14 11 08/30 Read block 504 len 32768 09/30 Read block 359 len 32768 10/30 Read block 290 len 8192 11/30 Read block 457 len 40960 12/30 Read block 547 len 40960 13/30 Write block 462 len 36864 data: 15 19 17 15 14 14 15 12 11 14/30 Read block 592 len 12288 15/30 Read block 216 len 20480 16/30 Read block 109 len 12288 17/30 Flush 18/30 Write block 352 len 4096 data: 8 19/30 Flush 20/30 Write block 328 len 28672 data: 12 8 11 14 14 12 12 21/30 Write block 166 len 20480 data: 8 12 15 13 14 22/30 Read block 575 len 20480 23/30 Read block 405 len 32768 24/30 Write block 258 len 40960 data: 12 11 13 15 14 16 15 16 15 17 25/30 Read block 233 len 40960 26/30 Read block 527 len 28672 27/30 Read block 205 len 28672 28/30 Read block 453 len 28672 29/30 Read block 67 len 32768 30/30 Write block 215 len 32768 data: 15 16 15 15 15 11 10 10 ---------------------------------------------------------------- Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Sent false 1001 Acked Read 5 Done Done Sent false 1002 Acked Read 4 Done Done Sent false 1003 Acked Read 7 Done Done Sent false 1004 Acked Flush 0 Done Done Sent false 1005 Acked Write 3 Done Done Sent false 1006 Acked Write 5 Done Done Sent false 1007 Acked Read 8 Done Done Sent false 1008 Acked Read 8 Done Done Sent false 1009 Acked Read 2 Done Done Sent false 1010 Acked Read 10 Done Done Sent false 1011 Acked Read 10 Done Done Sent false 1012 Acked Write 9 Done Done Sent false 1013 Acked Read 3 Done Done Sent false 1014 Acked Read 5 Done Done Sent false 1015 Acked Read 3 Done Done Sent false 1016 Acked Flush 0 Done Done Sent false 1017 Acked Write 1 Done Done Sent false 1018 Acked Flush 0 Done Done Sent false 1019 Acked Write 7 Done Done Sent false 1020 Acked Write 5 Done Done Sent false 1021 Acked Read 5 Done Done Sent false 1022 Acked Read 8 Done Done Sent false 1023 Acked Write 10 Done Done Sent false 1024 Acked Read 10 Done Done Sent false 1025 Acked Read 7 Done Done Sent false 1026 Acked Read 7 Done Done Sent false 1027 Acked Read 7 Done Done Sent false 1028 Acked Read 8 Done Done Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 30 32 Done 29 29 0 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:25:32.127858315Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1811} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:32.128041085Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1811,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 265 265 265 F F F 1 020-039 154 154 154 256 256 256 F F F 2 040-059 166 166 166 280 280 280 F F F 3 060-079 168 168 166 284 284 280 F F F <--- 4 080-099 166 166 166 280 280 280 F F F 5 100-119 166 166 166 279 279 279 F F F 6 120-139 167 167 167 282 282 282 F F F 7 140-159 168 168 167 283 283 282 F F F <--- 8 160-179 164 164 164 276 276 276 T T F <--- 9 180-199 166 166 166 280 280 280 F F F 10 200-219 166 166 166 281 281 281 T T F <--- 11 220-239 166 166 166 281 281 281 T T F <--- 12 240-259 168 168 162 284 284 272 T T F <--- 13 260-279 166 166 166 281 281 281 T T F <--- 14 280-299 164 164 164 275 275 275 F F F 15 300-319 166 166 166 280 280 280 F F F 16 320-339 158 158 158 265 265 265 T T F <--- 17 340-359 168 168 162 285 285 272 F F F <--- 18 360-379 164 164 164 276 276 276 F F F 19 380-399 164 164 164 276 276 276 F F F 20 400-419 167 167 167 282 282 282 F F F 21 420-439 165 165 165 278 278 278 F F F 22 440-459 158 158 158 264 264 264 F F F 23 460-479 168 168 165 284 284 278 F F F <--- 24 480-499 164 164 164 277 277 277 F F F 25 500-519 167 167 167 282 282 282 F F F 26 520-539 167 167 167 282 282 282 F F F 27 540-559 163 163 163 274 274 274 F F F 28 560-579 162 162 162 272 272 272 F F F 29 580-599 167 167 167 282 282 282 F F F Max gen: 168, Max flush: 285 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 84 Verifying data now No change after verify Loop: 84 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 265 265 265 F F F 1 020-039 154 154 154 256 256 256 F F F 2 040-059 166 166 166 280 280 280 F F F 3 060-079 168 168 168 284 284 284 F F F 4 080-099 166 166 166 280 280 280 F F F 5 100-119 166 166 166 279 279 279 F F F 6 120-139 167 167 167 282 282 282 F F F 7 140-159 168 168 168 283 283 283 F F F 8 160-179 169 169 169 286 286 286 F F F 9 180-199 166 166 166 280 280 280 F F F 10 200-219 169 169 169 286 286 286 F F F 11 220-239 169 169 169 286 286 286 F F F 12 240-259 169 169 169 286 286 286 F F F 13 260-279 169 169 169 286 286 286 F F F 14 280-299 164 164 164 275 275 275 F F F 15 300-319 166 166 166 280 280 280 F F F 16 320-339 169 169 169 286 286 286 F F F 17 340-359 168 168 168 285 285 285 F F F 18 360-379 164 164 164 276 276 276 F F F 19 380-399 164 164 164 276 276 276 F F F 20 400-419 167 167 167 282 282 282 F F F 21 420-439 165 165 165 278 278 278 F F F 22 440-459 158 158 158 264 264 264 F F F 23 460-479 168 168 168 284 284 284 F F F 24 480-499 164 164 164 277 277 277 F F F 25 500-519 167 167 167 282 282 282 F F F 26 520-539 167 167 167 282 282 282 F F F 27 540-559 163 163 163 274 274 274 F F F 28 560-579 162 162 162 272 272 272 F F F 29 580-599 167 167 167 282 282 282 F F F Max gen: 169, Max flush: 286 Begin loop 85 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:36.374964814Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1819,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:36.383116406Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1819,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:36.383422936Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1819} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:36.383472297Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1819} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:36.383498647Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1819} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:36.383511607Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1819} {"msg":"Crucible d56ed5af-3dcf-48ee-9311-177e028fed76 has session id: bb5ac7a9-a69d-48ca-a8ba-1423ee3cbe3e","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:36.383813627Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1819,"session_id":"bb5ac7a9-a69d-48ca-a8ba-1423ee3cbe3e"} {"msg":"Upstairs opts: Upstairs UUID: d56ed5af-3dcf-48ee-9311-177e028fed76, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:36.383837277Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1819,"session_id":"bb5ac7a9-a69d-48ca-a8ba-1423ee3cbe3e"} {"msg":"Crucible stats registered with UUID: d56ed5af-3dcf-48ee-9311-177e028fed76","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:36.383847767Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1819,"session_id":"bb5ac7a9-a69d-48ca-a8ba-1423ee3cbe3e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:25:36.383857077Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1819,"session_id":"bb5ac7a9-a69d-48ca-a8ba-1423ee3cbe3e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:36.384008427Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1819,"":"io task","client":"1","":"downstairs","session_id":"bb5ac7a9-a69d-48ca-a8ba-1423ee3cbe3e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:36.384039757Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1819,"":"io task","client":"0","":"downstairs","session_id":"bb5ac7a9-a69d-48ca-a8ba-1423ee3cbe3e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:36.384052837Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1819,"":"io task","client":"2","":"downstairs","session_id":"bb5ac7a9-a69d-48ca-a8ba-1423ee3cbe3e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:36.384176548Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1819,"task":"crutest"} {"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:36.384188768Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1819} {"msg":"d56ed5af-3dcf-48ee-9311-177e028fed76 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:36.384222228Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1819,"session_id":"bb5ac7a9-a69d-48ca-a8ba-1423ee3cbe3e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:36.384231718Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1819,"client":"0","":"downstairs","session_id":"bb5ac7a9-a69d-48ca-a8ba-1423ee3cbe3e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:36.384241518Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1819,"client":"1","":"downstairs","session_id":"bb5ac7a9-a69d-48ca-a8ba-1423ee3cbe3e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:36.384251318Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1819,"client":"2","":"downstairs","session_id":"bb5ac7a9-a69d-48ca-a8ba-1423ee3cbe3e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:36.384260988Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1819,"":"io task","client":"2","":"downstairs","session_id":"bb5ac7a9-a69d-48ca-a8ba-1423ee3cbe3e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:36.384271188Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1819,"":"io task","client":"0","":"downstairs","session_id":"bb5ac7a9-a69d-48ca-a8ba-1423ee3cbe3e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:36.384308548Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1819,"":"io task","client":"1","":"downstairs","session_id":"bb5ac7a9-a69d-48ca-a8ba-1423ee3cbe3e"} {"msg":"ds_connection connected from Ok(127.0.0.1:57454)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:36.384340238Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1819,"":"io task","client":"0","":"downstairs","session_id":"bb5ac7a9-a69d-48ca-a8ba-1423ee3cbe3e"} {"msg":"ds_connection connected from Ok(127.0.0.1:36406)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:36.384351448Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1819,"":"io task","client":"2","":"downstairs","session_id":"bb5ac7a9-a69d-48ca-a8ba-1423ee3cbe3e"} {"msg":"ds_connection connected from Ok(127.0.0.1:56768)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:36.384362018Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1819,"":"io task","client":"1","":"downstairs","session_id":"bb5ac7a9-a69d-48ca-a8ba-1423ee3cbe3e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:36.384922069Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1819,"client":"0","":"downstairs","session_id":"bb5ac7a9-a69d-48ca-a8ba-1423ee3cbe3e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:36.384945779Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1819,"client":"1","":"downstairs","session_id":"bb5ac7a9-a69d-48ca-a8ba-1423ee3cbe3e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:36.385080609Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1819,"client":"2","":"downstairs","session_id":"bb5ac7a9-a69d-48ca-a8ba-1423ee3cbe3e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:36.385221099Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1819,"session_id":"bb5ac7a9-a69d-48ca-a8ba-1423ee3cbe3e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:36.385256039Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1819,"session_id":"bb5ac7a9-a69d-48ca-a8ba-1423ee3cbe3e"} {"msg":"[0]R flush_numbers[0..12]: [265, 256, 280, 284, 280, 279, 282, 283, 286, 280, 286, 286]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:36.385348449Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1819,"":"downstairs","":"downstairs","session_id":"bb5ac7a9-a69d-48ca-a8ba-1423ee3cbe3e"} {"msg":"[0]R generation[0..12]: [158, 154, 166, 168, 166, 166, 167, 168, 169, 166, 169, 169]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:36.385360009Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1819,"":"downstairs","":"downstairs","session_id":"bb5ac7a9-a69d-48ca-a8ba-1423ee3cbe3e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:36.385371009Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1819,"":"downstairs","":"downstairs","session_id":"bb5ac7a9-a69d-48ca-a8ba-1423ee3cbe3e"} {"msg":"[1]R flush_numbers[0..12]: [265, 256, 280, 284, 280, 279, 282, 283, 286, 280, 286, 286]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:36.385381749Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1819,"":"downstairs","":"downstairs","session_id":"bb5ac7a9-a69d-48ca-a8ba-1423ee3cbe3e"} {"msg":"[1]R generation[0..12]: [158, 154, 166, 168, 166, 166, 167, 168, 169, 166, 169, 169]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:36.385392589Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1819,"":"downstairs","":"downstairs","session_id":"bb5ac7a9-a69d-48ca-a8ba-1423ee3cbe3e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:36.385403569Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1819,"":"downstairs","":"downstairs","session_id":"bb5ac7a9-a69d-48ca-a8ba-1423ee3cbe3e"} {"msg":"[2]R flush_numbers[0..12]: [265, 256, 280, 284, 280, 279, 282, 283, 286, 280, 286, 286]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:36.385414379Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1819,"":"downstairs","":"downstairs","session_id":"bb5ac7a9-a69d-48ca-a8ba-1423ee3cbe3e"} {"msg":"[2]R generation[0..12]: [158, 154, 166, 168, 166, 166, 167, 168, 169, 166, 169, 169]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:36.38542515Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1819,"":"downstairs","":"downstairs","session_id":"bb5ac7a9-a69d-48ca-a8ba-1423ee3cbe3e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:36.38543639Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1819,"":"downstairs","":"downstairs","session_id":"bb5ac7a9-a69d-48ca-a8ba-1423ee3cbe3e"} {"msg":"Max found gen is 170","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:36.38544587Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1819,"":"downstairs","":"downstairs","session_id":"bb5ac7a9-a69d-48ca-a8ba-1423ee3cbe3e"} {"msg":"Generation requested: 170 >= found:170","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:36.38545575Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1819,"":"downstairs","":"downstairs","session_id":"bb5ac7a9-a69d-48ca-a8ba-1423ee3cbe3e"} {"msg":"Next flush: 287","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:36.38546532Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1819,"":"downstairs","":"downstairs","session_id":"bb5ac7a9-a69d-48ca-a8ba-1423ee3cbe3e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:36.38547622Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1819,"":"downstairs","":"downstairs","session_id":"bb5ac7a9-a69d-48ca-a8ba-1423ee3cbe3e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:36.38548605Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1819,"session_id":"bb5ac7a9-a69d-48ca-a8ba-1423ee3cbe3e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:36.38549488Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1819,"client":"0","":"downstairs","session_id":"bb5ac7a9-a69d-48ca-a8ba-1423ee3cbe3e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:36.38550476Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1819,"client":"1","":"downstairs","session_id":"bb5ac7a9-a69d-48ca-a8ba-1423ee3cbe3e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:36.38551448Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1819,"client":"2","":"downstairs","session_id":"bb5ac7a9-a69d-48ca-a8ba-1423ee3cbe3e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:36.38552438Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1819,"session_id":"bb5ac7a9-a69d-48ca-a8ba-1423ee3cbe3e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:36.38553358Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1819,"session_id":"bb5ac7a9-a69d-48ca-a8ba-1423ee3cbe3e"} {"msg":"d56ed5af-3dcf-48ee-9311-177e028fed76 is now active with session: bb5ac7a9-a69d-48ca-a8ba-1423ee3cbe3e","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:36.3855437Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1819,"session_id":"bb5ac7a9-a69d-48ca-a8ba-1423ee3cbe3e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:36.38555275Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1819,"session_id":"bb5ac7a9-a69d-48ca-a8ba-1423ee3cbe3e"} {"msg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:36.38556194Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1819} {"msg":"Activated sub_volume d56ed5af-3dcf-48ee-9311-177e028fed76","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:36.3855701Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1819} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:36.38557824Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1819,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:36.38589771Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1819,"task":"crutest"} 01/30 Write block 1 len 8192 data: 4 5 02/30 Write block 451 len 28672 data: 7 8 8 9 12 15 11 03/30 Read block 86 len 24576 04/30 Write block 560 len 20480 data: 11 12 11 12 10 05/30 Read block 260 len 16384 06/30 Read block 104 len 24576 07/30 Write block 10 len 32768 data: 11 10 12 10 12 11 11 11 08/30 Flush 09/30 Write block 566 len 4096 data: 13 10/30 Write block 388 len 24576 data: 12 11 11 11 12 9 11/30 Read block 482 len 40960 12/30 Write block 109 len 28672 data: 12 13 14 14 16 15 15 13/30 Read block 5 len 20480 14/30 Read block 422 len 20480 15/30 Write block 565 len 16384 data: 13 14 16 14 16/30 Read block 179 len 12288 17/30 Flush 18/30 Write block 496 len 40960 data: 12 11 10 14 12 12 15 13 10 12 19/30 Flush 20/30 Write block 128 len 20480 data: 10 9 7 9 10 21/30 Read block 312 len 36864 22/30 Flush 23/30 Write block 280 len 24576 data: 15 12 11 10 8 10 24/30 Flush 25/30 Write block 47 len 16384 data: 8 9 9 13 26/30 Read block 290 len 4096 27/30 Read block 22 len 24576 28/30 Read block 77 len 16384 29/30 Read block 15 len 4096 30/30 Write block 417 len 20480 data: 18 14 12 13 13 ---------------------------------------------------------------- Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Write 7 Done Done Done false 1002 Acked Read 6 Done Done Done false 1003 Acked Write 5 Done Done Sent false 1004 Acked Read 4 Done Done Sent false 1005 Acked Read 6 Done Done Sent false 1006 Acked Write 8 Done Done Sent false 1007 Acked Flush 0 Done Done Sent false 1008 Acked Write 1 Done Done Sent false 1009 Acked Write 6 Done Done Sent false 1010 Acked Read 10 Done Done Sent false 1011 Acked Write 7 Done Done Sent false 1012 Acked Read 5 Done Done Sent false 1013 Acked Read 5 Done Done Sent false 1014 Acked Write 4 Done Done Sent false 1015 Acked Read 3 Done Done Sent false 1016 Acked Flush 0 Done Done Sent false 1017 Acked Write 10 Done Done Sent false 1018 Acked Flush 0 Done Done Sent false 1019 Acked Write 5 Done Done Sent false 1020 Acked Read 9 Done Done Sent false 1021 Acked Flush 0 Done Done Sent false 1022 Acked Write 6 Done Done Sent false 1023 Acked Flush 0 Done Done Sent false 1024 Acked Write 4 Done Done Sent false 1025 Acked Read 1 Done Done Sent false 1026 Acked Read 6 Done Done Sent false 1027 Acked Read 4 Done Done Sent false 1028 Acked Read 1 Done Done Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 27 29 Done 29 29 3 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:25:36.394785664Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1819} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:36.394976304Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1819,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 170 170 158 287 287 265 F F T <--- 1 020-039 154 154 154 256 256 256 F F F 2 040-059 166 166 166 280 280 280 T T F <--- 3 060-079 168 168 168 284 284 284 F F F 4 080-099 166 166 166 280 280 280 F F F 5 100-119 170 170 166 288 288 279 F F F <--- 6 120-139 170 170 167 290 290 282 F F F <--- 7 140-159 168 168 168 283 283 283 F F F 8 160-179 169 169 169 286 286 286 F F F 9 180-199 166 166 166 280 280 280 F F F 10 200-219 169 169 169 286 286 286 F F F 11 220-239 169 169 169 286 286 286 F F F 12 240-259 169 169 169 286 286 286 F F F 13 260-279 169 169 169 286 286 286 F F F 14 280-299 170 170 164 291 291 275 F F F <--- 15 300-319 166 166 166 280 280 280 F F F 16 320-339 169 169 169 286 286 286 F F F 17 340-359 168 168 168 285 285 285 F F F 18 360-379 164 164 164 276 276 276 F F F 19 380-399 170 170 164 288 288 276 F F F <--- 20 400-419 167 167 167 282 282 282 T T F <--- 21 420-439 165 165 165 278 278 278 T T F <--- 22 440-459 170 170 158 287 287 264 F F T <--- 23 460-479 168 168 168 284 284 284 F F F 24 480-499 170 170 164 289 289 277 F F F <--- 25 500-519 170 170 167 289 289 282 F F F <--- 26 520-539 167 167 167 282 282 282 F F F 27 540-559 163 163 163 274 274 274 F F F 28 560-579 170 170 162 288 288 272 F F F <--- 29 580-599 167 167 167 282 282 282 F F F Max gen: 170, Max flush: 291 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 85 Verifying data now No change after verify Loop: 85 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 170 170 170 287 287 287 F F F 1 020-039 154 154 154 256 256 256 F F F 2 040-059 171 171 171 292 292 292 F F F 3 060-079 168 168 168 284 284 284 F F F 4 080-099 166 166 166 280 280 280 F F F 5 100-119 170 170 170 288 288 288 F F F 6 120-139 170 170 170 290 290 290 F F F 7 140-159 168 168 168 283 283 283 F F F 8 160-179 169 169 169 286 286 286 F F F 9 180-199 166 166 166 280 280 280 F F F 10 200-219 169 169 169 286 286 286 F F F 11 220-239 169 169 169 286 286 286 F F F 12 240-259 169 169 169 286 286 286 F F F 13 260-279 169 169 169 286 286 286 F F F 14 280-299 170 170 170 291 291 291 F F F 15 300-319 166 166 166 280 280 280 F F F 16 320-339 169 169 169 286 286 286 F F F 17 340-359 168 168 168 285 285 285 F F F 18 360-379 164 164 164 276 276 276 F F F 19 380-399 170 170 170 288 288 288 F F F 20 400-419 171 171 171 292 292 292 F F F 21 420-439 171 171 171 292 292 292 F F F 22 440-459 170 170 170 287 287 287 F F F 23 460-479 168 168 168 284 284 284 F F F 24 480-499 170 170 170 289 289 289 F F F 25 500-519 170 170 170 289 289 289 F F F 26 520-539 167 167 167 282 282 282 F F F 27 540-559 163 163 163 274 274 274 F F F 28 560-579 170 170 170 288 288 288 F F F 29 580-599 167 167 167 282 282 282 F F F Max gen: 171, Max flush: 292 Begin loop 86 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:40.645779039Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1827,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:25:40.652914429Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1827,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:40.652975019Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1827,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:40.659638239Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1827,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:40.6599232Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1827} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:40.65994858Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1827} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:40.65995722Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1827} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:40.6599692Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1827} {"msg":"Crucible 78e687a9-ee8b-4710-b2a1-2e1c341d4d7f has session id: 5991f60d-1615-4e72-9ab6-8c7f209629ee","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:40.66030176Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1827,"session_id":"5991f60d-1615-4e72-9ab6-8c7f209629ee"} {"msg":"Upstairs opts: Upstairs UUID: 78e687a9-ee8b-4710-b2a1-2e1c341d4d7f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:40.66033122Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1827,"session_id":"5991f60d-1615-4e72-9ab6-8c7f209629ee"} {"msg":"Crucible stats registered with UUID: 78e687a9-ee8b-4710-b2a1-2e1c341d4d7f","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:40.6603423Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1827,"session_id":"5991f60d-1615-4e72-9ab6-8c7f209629ee"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:25:40.660351861Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1827,"session_id":"5991f60d-1615-4e72-9ab6-8c7f209629ee"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:40.660468931Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1827,"":"io task","client":"2","":"downstairs","session_id":"5991f60d-1615-4e72-9ab6-8c7f209629ee"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:40.660500871Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1827,"":"io task","client":"0","":"downstairs","session_id":"5991f60d-1615-4e72-9ab6-8c7f209629ee"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:40.660512091Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1827,"":"io task","client":"1","":"downstairs","session_id":"5991f60d-1615-4e72-9ab6-8c7f209629ee"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:40.660655301Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1827,"task":"crutest"} {"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:40.660673891Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1827} {"msg":"78e687a9-ee8b-4710-b2a1-2e1c341d4d7f active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:40.660704501Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1827,"session_id":"5991f60d-1615-4e72-9ab6-8c7f209629ee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:40.660716091Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1827,"client":"0","":"downstairs","session_id":"5991f60d-1615-4e72-9ab6-8c7f209629ee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:40.660725961Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1827,"client":"1","":"downstairs","session_id":"5991f60d-1615-4e72-9ab6-8c7f209629ee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:40.660735851Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1827,"client":"2","":"downstairs","session_id":"5991f60d-1615-4e72-9ab6-8c7f209629ee"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:40.660745481Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1827,"":"io task","client":"2","":"downstairs","session_id":"5991f60d-1615-4e72-9ab6-8c7f209629ee"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:40.660755401Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1827,"":"io task","client":"0","":"downstairs","session_id":"5991f60d-1615-4e72-9ab6-8c7f209629ee"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:40.660765411Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1827,"":"io task","client":"1","":"downstairs","session_id":"5991f60d-1615-4e72-9ab6-8c7f209629ee"} {"msg":"ds_connection connected from Ok(127.0.0.1:44565)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:40.660790981Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1827,"":"io task","client":"0","":"downstairs","session_id":"5991f60d-1615-4e72-9ab6-8c7f209629ee"} {"msg":"ds_connection connected from Ok(127.0.0.1:52667)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:40.660809691Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1827,"":"io task","client":"2","":"downstairs","session_id":"5991f60d-1615-4e72-9ab6-8c7f209629ee"} {"msg":"ds_connection connected from Ok(127.0.0.1:61902)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:40.660953731Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1827,"":"io task","client":"1","":"downstairs","session_id":"5991f60d-1615-4e72-9ab6-8c7f209629ee"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:40.661402742Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1827,"client":"1","":"downstairs","session_id":"5991f60d-1615-4e72-9ab6-8c7f209629ee"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:40.661477692Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1827,"client":"2","":"downstairs","session_id":"5991f60d-1615-4e72-9ab6-8c7f209629ee"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:40.661504672Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1827,"client":"0","":"downstairs","session_id":"5991f60d-1615-4e72-9ab6-8c7f209629ee"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:40.661705043Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1827,"session_id":"5991f60d-1615-4e72-9ab6-8c7f209629ee"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:40.661754583Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1827,"session_id":"5991f60d-1615-4e72-9ab6-8c7f209629ee"} {"msg":"[0]R flush_numbers[0..12]: [287, 256, 292, 284, 280, 288, 290, 283, 286, 280, 286, 286]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:40.661795063Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1827,"":"downstairs","":"downstairs","session_id":"5991f60d-1615-4e72-9ab6-8c7f209629ee"} {"msg":"[0]R generation[0..12]: [170, 154, 171, 168, 166, 170, 170, 168, 169, 166, 169, 169]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:40.661810193Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1827,"":"downstairs","":"downstairs","session_id":"5991f60d-1615-4e72-9ab6-8c7f209629ee"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:40.661821753Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1827,"":"downstairs","":"downstairs","session_id":"5991f60d-1615-4e72-9ab6-8c7f209629ee"} {"msg":"[1]R flush_numbers[0..12]: [287, 256, 292, 284, 280, 288, 290, 283, 286, 280, 286, 286]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:40.661832573Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1827,"":"downstairs","":"downstairs","session_id":"5991f60d-1615-4e72-9ab6-8c7f209629ee"} {"msg":"[1]R generation[0..12]: [170, 154, 171, 168, 166, 170, 170, 168, 169, 166, 169, 169]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:40.661843523Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1827,"":"downstairs","":"downstairs","session_id":"5991f60d-1615-4e72-9ab6-8c7f209629ee"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:40.661854663Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1827,"":"downstairs","":"downstairs","session_id":"5991f60d-1615-4e72-9ab6-8c7f209629ee"} {"msg":"[2]R flush_numbers[0..12]: [287, 256, 292, 284, 280, 288, 290, 283, 286, 280, 286, 286]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:40.661865473Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1827,"":"downstairs","":"downstairs","session_id":"5991f60d-1615-4e72-9ab6-8c7f209629ee"} {"msg":"[2]R generation[0..12]: [170, 154, 171, 168, 166, 170, 170, 168, 169, 166, 169, 169]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:40.661876183Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1827,"":"downstairs","":"downstairs","session_id":"5991f60d-1615-4e72-9ab6-8c7f209629ee"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:40.661887213Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1827,"":"downstairs","":"downstairs","session_id":"5991f60d-1615-4e72-9ab6-8c7f209629ee"} {"msg":"Max found gen is 172","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:40.661896823Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1827,"":"downstairs","":"downstairs","session_id":"5991f60d-1615-4e72-9ab6-8c7f209629ee"} {"msg":"Generation requested: 172 >= found:172","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:40.661906613Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1827,"":"downstairs","":"downstairs","session_id":"5991f60d-1615-4e72-9ab6-8c7f209629ee"} {"msg":"Next flush: 293","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:40.661916203Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1827,"":"downstairs","":"downstairs","session_id":"5991f60d-1615-4e72-9ab6-8c7f209629ee"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:40.661925773Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1827,"":"downstairs","":"downstairs","session_id":"5991f60d-1615-4e72-9ab6-8c7f209629ee"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:40.661935563Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1827,"session_id":"5991f60d-1615-4e72-9ab6-8c7f209629ee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:40.661944333Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1827,"client":"0","":"downstairs","session_id":"5991f60d-1615-4e72-9ab6-8c7f209629ee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:40.661954183Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1827,"client":"1","":"downstairs","session_id":"5991f60d-1615-4e72-9ab6-8c7f209629ee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:40.661965323Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1827,"client":"2","":"downstairs","session_id":"5991f60d-1615-4e72-9ab6-8c7f209629ee"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:40.661975463Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1827,"session_id":"5991f60d-1615-4e72-9ab6-8c7f209629ee"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:40.661984723Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1827,"session_id":"5991f60d-1615-4e72-9ab6-8c7f209629ee"} {"msg":"78e687a9-ee8b-4710-b2a1-2e1c341d4d7f is now active with session: 5991f60d-1615-4e72-9ab6-8c7f209629ee","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:40.661994893Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1827,"session_id":"5991f60d-1615-4e72-9ab6-8c7f209629ee"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:40.662015323Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1827,"session_id":"5991f60d-1615-4e72-9ab6-8c7f209629ee"} {"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:40.662024473Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1827} {"msg":"Activated sub_volume 78e687a9-ee8b-4710-b2a1-2e1c341d4d7f","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:40.662032543Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1827} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:40.662040833Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1827,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:40.662325033Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1827,"task":"crutest"} 01/30 Write block 480 len 28672 data: 9 8 8 10 12 14 16 02/30 Write block 66 len 24576 data: 14 14 11 13 15 12 03/30 Read block 359 len 24576 04/30 Flush 05/30 Write block 579 len 24576 data: 13 13 11 12 14 16 06/30 Write block 113 len 16384 data: 17 16 16 15 07/30 Write block 402 len 28672 data: 9 7 9 10 8 11 12 08/30 Read block 410 len 40960 09/30 Write block 223 len 12288 data: 12 9 11 10/30 Flush 11/30 Write block 201 len 40960 data: 10 11 14 13 11 10 10 13 14 14 12/30 Read block 579 len 24576 13/30 Read block 242 len 36864 14/30 Flush 15/30 Write block 285 len 40960 data: 11 11 10 9 10 11 9 11 14 15 16/30 Write block 19 len 32768 data: 14 15 16 15 17 16 15 13 17/30 Read block 575 len 36864 18/30 Write block 177 len 36864 data: 11 14 12 11 10 8 10 12 16 19/30 Read block 21 len 40960 20/30 Write block 276 len 36864 data: 17 18 13 12 16 13 12 11 9 21/30 Read block 562 len 32768 22/30 Read block 71 len 16384 23/30 Write block 431 len 4096 data: 9 24/30 Write block 288 len 36864 data: 10 11 12 10 12 15 16 16 17 25/30 Read block 471 len 20480 26/30 Write block 121 len 40960 data: 12 13 12 11 13 11 9 11 10 8 27/30 Read block 273 len 8192 28/30 Read block 220 len 24576 29/30 Write block 237 len 8192 data: 17 16 30/30 Read block 289 len 32768 ---------------------------------------------------------------- Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Write 6 Done Done Done false 1002 Acked Read 6 Sent Done Done false 1003 Acked Flush 0 Sent Done Done false 1004 Acked Write 6 Sent Done Done false 1005 Acked Write 4 Sent Done Done false 1006 Acked Write 7 Sent Done Done false 1007 Acked Read 10 Sent Done Done false 1008 Acked Write 3 Sent Done Done false 1009 Acked Flush 0 Sent Done Done false 1010 Acked Write 10 Sent Done Done false 1011 Acked Read 6 Sent Done Done false 1012 Acked Read 9 Sent Done Done false 1013 Acked Flush 0 Sent Done Done false 1014 Acked Write 10 Sent Done Done false 1015 Acked Write 8 Sent Done Done false 1016 Acked Read 9 Sent Done Done false 1017 Acked Write 9 Sent Done Done false 1018 Acked Read 10 Sent Done Done false 1019 Acked Write 9 Sent Done Done false 1020 Acked Read 8 Sent Done Done false 1021 Acked Read 4 Sent Done Done false 1022 Acked Write 1 Sent Done Done false 1023 Acked Write 9 Sent Done Done false 1024 Acked Read 5 Sent Done Done false 1025 Acked Write 10 Sent Done Done false 1026 Acked Read 2 Sent Done Done false 1027 Acked Read 6 Sent Done Done false 1028 Acked Write 2 Sent Done Done false 1029 Acked Read 8 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 0 1 29 Done 2 30 29 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1013)) Some(JobId(1013)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:25:40.672250378Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1827} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:40.672436749Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1827,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 170 170 170 287 287 287 F T T <--- 1 020-039 154 154 154 256 256 256 F T T <--- 2 040-059 171 171 171 292 292 292 F F F 3 060-079 168 172 172 284 293 293 T F F <--- 4 080-099 166 166 166 280 280 280 F F F 5 100-119 170 172 172 288 294 294 F F F <--- 6 120-139 170 170 170 290 290 290 F T T <--- 7 140-159 168 168 168 283 283 283 F F F 8 160-179 169 169 169 286 286 286 F T T <--- 9 180-199 166 166 166 280 280 280 F T T <--- 10 200-219 169 172 172 286 295 295 F F F <--- 11 220-239 169 172 172 286 294 294 F T T <--- 12 240-259 169 169 169 286 286 286 F F F 13 260-279 169 169 169 286 286 286 F T T <--- 14 280-299 170 170 170 291 291 291 F T T <--- 15 300-319 166 166 166 280 280 280 F F F 16 320-339 169 169 169 286 286 286 F F F 17 340-359 168 168 168 285 285 285 F F F 18 360-379 164 164 164 276 276 276 F F F 19 380-399 170 170 170 288 288 288 F F F 20 400-419 171 172 172 292 294 294 F F F <--- 21 420-439 171 171 171 292 292 292 F T T <--- 22 440-459 170 170 170 287 287 287 F F F 23 460-479 168 168 168 284 284 284 F F F 24 480-499 170 172 172 289 293 293 T F F <--- 25 500-519 170 170 170 289 289 289 F F F 26 520-539 167 167 167 282 282 282 F F F 27 540-559 163 163 163 274 274 274 F F F 28 560-579 170 172 172 288 294 294 F F F <--- 29 580-599 167 172 172 282 294 294 F F F <--- Max gen: 172, Max flush: 295 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 86 Verifying data now No change after verify Loop: 86 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 173 173 173 296 296 296 F F F 1 020-039 173 173 173 296 296 296 F F F 2 040-059 171 171 171 292 292 292 F F F 3 060-079 172 172 172 293 293 293 F F F 4 080-099 166 166 166 280 280 280 F F F 5 100-119 172 172 172 294 294 294 F F F 6 120-139 173 173 173 296 296 296 F F F 7 140-159 168 168 168 283 283 283 F F F 8 160-179 173 173 173 296 296 296 F F F 9 180-199 173 173 173 296 296 296 F F F 10 200-219 172 172 172 295 295 295 F F F 11 220-239 173 173 173 296 296 296 F F F 12 240-259 169 169 169 286 286 286 F F F 13 260-279 173 173 173 296 296 296 F F F 14 280-299 173 173 173 296 296 296 F F F 15 300-319 166 166 166 280 280 280 F F F 16 320-339 169 169 169 286 286 286 F F F 17 340-359 168 168 168 285 285 285 F F F 18 360-379 164 164 164 276 276 276 F F F 19 380-399 170 170 170 288 288 288 F F F 20 400-419 172 172 172 294 294 294 F F F 21 420-439 173 173 173 296 296 296 F F F 22 440-459 170 170 170 287 287 287 F F F 23 460-479 168 168 168 284 284 284 F F F 24 480-499 172 172 172 293 293 293 F F F 25 500-519 170 170 170 289 289 289 F F F 26 520-539 167 167 167 282 282 282 F F F 27 540-559 163 163 163 274 274 274 F F F 28 560-579 172 172 172 294 294 294 F F F 29 580-599 172 172 172 294 294 294 F F F Max gen: 173, Max flush: 296 Begin loop 87 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:44.938996467Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1835,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:25:44.946250298Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1835,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:44.946313018Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1835,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:44.953267508Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1835,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:44.953545819Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1835} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:44.953575329Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1835} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:44.953586449Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1835} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:44.953598299Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1835} {"msg":"Crucible 0aadeeeb-2e52-442c-8837-85eda8a01852 has session id: 5b5afc93-3d1f-4ae6-b7e6-35ff8890b74a","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:44.953894869Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1835,"session_id":"5b5afc93-3d1f-4ae6-b7e6-35ff8890b74a"} {"msg":"Upstairs opts: Upstairs UUID: 0aadeeeb-2e52-442c-8837-85eda8a01852, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:44.953923569Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1835,"session_id":"5b5afc93-3d1f-4ae6-b7e6-35ff8890b74a"} {"msg":"Crucible stats registered with UUID: 0aadeeeb-2e52-442c-8837-85eda8a01852","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:44.953934569Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1835,"session_id":"5b5afc93-3d1f-4ae6-b7e6-35ff8890b74a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:25:44.953943799Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1835,"session_id":"5b5afc93-3d1f-4ae6-b7e6-35ff8890b74a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:44.954067519Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1835,"":"io task","client":"0","":"downstairs","session_id":"5b5afc93-3d1f-4ae6-b7e6-35ff8890b74a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:44.954100109Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1835,"":"io task","client":"1","":"downstairs","session_id":"5b5afc93-3d1f-4ae6-b7e6-35ff8890b74a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:44.954111359Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1835,"":"io task","client":"2","":"downstairs","session_id":"5b5afc93-3d1f-4ae6-b7e6-35ff8890b74a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:44.95425279Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1835,"task":"crutest"} {"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:44.9542722Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1835} {"msg":"0aadeeeb-2e52-442c-8837-85eda8a01852 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:44.95428841Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1835,"session_id":"5b5afc93-3d1f-4ae6-b7e6-35ff8890b74a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:44.95429753Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1835,"client":"0","":"downstairs","session_id":"5b5afc93-3d1f-4ae6-b7e6-35ff8890b74a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:44.95430721Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1835,"client":"1","":"downstairs","session_id":"5b5afc93-3d1f-4ae6-b7e6-35ff8890b74a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:44.95431691Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1835,"client":"2","":"downstairs","session_id":"5b5afc93-3d1f-4ae6-b7e6-35ff8890b74a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:44.95432861Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1835,"":"io task","client":"2","":"downstairs","session_id":"5b5afc93-3d1f-4ae6-b7e6-35ff8890b74a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:44.95433875Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1835,"":"io task","client":"1","":"downstairs","session_id":"5b5afc93-3d1f-4ae6-b7e6-35ff8890b74a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:44.95437308Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1835,"":"io task","client":"0","":"downstairs","session_id":"5b5afc93-3d1f-4ae6-b7e6-35ff8890b74a"} {"msg":"ds_connection connected from Ok(127.0.0.1:50394)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:44.95438871Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1835,"":"io task","client":"1","":"downstairs","session_id":"5b5afc93-3d1f-4ae6-b7e6-35ff8890b74a"} {"msg":"ds_connection connected from Ok(127.0.0.1:46392)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:44.95439901Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1835,"":"io task","client":"2","":"downstairs","session_id":"5b5afc93-3d1f-4ae6-b7e6-35ff8890b74a"} {"msg":"ds_connection connected from Ok(127.0.0.1:50418)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:44.95449768Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1835,"":"io task","client":"0","":"downstairs","session_id":"5b5afc93-3d1f-4ae6-b7e6-35ff8890b74a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:44.955072011Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1835,"client":"2","":"downstairs","session_id":"5b5afc93-3d1f-4ae6-b7e6-35ff8890b74a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:44.955084691Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1835,"client":"0","":"downstairs","session_id":"5b5afc93-3d1f-4ae6-b7e6-35ff8890b74a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:44.955095661Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1835,"client":"1","":"downstairs","session_id":"5b5afc93-3d1f-4ae6-b7e6-35ff8890b74a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:44.955415931Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1835,"session_id":"5b5afc93-3d1f-4ae6-b7e6-35ff8890b74a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:44.955426731Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1835,"session_id":"5b5afc93-3d1f-4ae6-b7e6-35ff8890b74a"} {"msg":"[0]R flush_numbers[0..12]: [296, 296, 292, 293, 280, 294, 296, 283, 296, 296, 295, 296]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:44.955460861Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1835,"":"downstairs","":"downstairs","session_id":"5b5afc93-3d1f-4ae6-b7e6-35ff8890b74a"} {"msg":"[0]R generation[0..12]: [173, 173, 171, 172, 166, 172, 173, 168, 173, 173, 172, 173]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:44.955472172Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1835,"":"downstairs","":"downstairs","session_id":"5b5afc93-3d1f-4ae6-b7e6-35ff8890b74a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:44.955483322Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1835,"":"downstairs","":"downstairs","session_id":"5b5afc93-3d1f-4ae6-b7e6-35ff8890b74a"} {"msg":"[1]R flush_numbers[0..12]: [296, 296, 292, 293, 280, 294, 296, 283, 296, 296, 295, 296]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:44.955494092Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1835,"":"downstairs","":"downstairs","session_id":"5b5afc93-3d1f-4ae6-b7e6-35ff8890b74a"} {"msg":"[1]R generation[0..12]: [173, 173, 171, 172, 166, 172, 173, 168, 173, 173, 172, 173]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:44.955506952Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1835,"":"downstairs","":"downstairs","session_id":"5b5afc93-3d1f-4ae6-b7e6-35ff8890b74a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:44.955518042Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1835,"":"downstairs","":"downstairs","session_id":"5b5afc93-3d1f-4ae6-b7e6-35ff8890b74a"} {"msg":"[2]R flush_numbers[0..12]: [296, 296, 292, 293, 280, 294, 296, 283, 296, 296, 295, 296]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:44.955528832Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1835,"":"downstairs","":"downstairs","session_id":"5b5afc93-3d1f-4ae6-b7e6-35ff8890b74a"} {"msg":"[2]R generation[0..12]: [173, 173, 171, 172, 166, 172, 173, 168, 173, 173, 172, 173]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:44.955539632Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1835,"":"downstairs","":"downstairs","session_id":"5b5afc93-3d1f-4ae6-b7e6-35ff8890b74a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:44.955550632Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1835,"":"downstairs","":"downstairs","session_id":"5b5afc93-3d1f-4ae6-b7e6-35ff8890b74a"} {"msg":"Max found gen is 174","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:44.955560292Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1835,"":"downstairs","":"downstairs","session_id":"5b5afc93-3d1f-4ae6-b7e6-35ff8890b74a"} {"msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:44.955570182Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1835,"":"downstairs","":"downstairs","session_id":"5b5afc93-3d1f-4ae6-b7e6-35ff8890b74a"} {"msg":"Next flush: 297","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:44.955579602Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1835,"":"downstairs","":"downstairs","session_id":"5b5afc93-3d1f-4ae6-b7e6-35ff8890b74a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:44.955588982Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1835,"":"downstairs","":"downstairs","session_id":"5b5afc93-3d1f-4ae6-b7e6-35ff8890b74a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:44.955599072Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1835,"session_id":"5b5afc93-3d1f-4ae6-b7e6-35ff8890b74a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:44.955608092Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1835,"client":"0","":"downstairs","session_id":"5b5afc93-3d1f-4ae6-b7e6-35ff8890b74a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:44.955618172Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1835,"client":"1","":"downstairs","session_id":"5b5afc93-3d1f-4ae6-b7e6-35ff8890b74a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:44.955628002Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1835,"client":"2","":"downstairs","session_id":"5b5afc93-3d1f-4ae6-b7e6-35ff8890b74a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:44.955638092Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1835,"session_id":"5b5afc93-3d1f-4ae6-b7e6-35ff8890b74a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:44.955647392Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1835,"session_id":"5b5afc93-3d1f-4ae6-b7e6-35ff8890b74a"} {"msg":"0aadeeeb-2e52-442c-8837-85eda8a01852 is now active with session: 5b5afc93-3d1f-4ae6-b7e6-35ff8890b74a","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:44.955658722Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1835,"session_id":"5b5afc93-3d1f-4ae6-b7e6-35ff8890b74a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:44.955667522Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1835,"session_id":"5b5afc93-3d1f-4ae6-b7e6-35ff8890b74a"} {"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:44.955676792Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1835} {"msg":"Activated sub_volume 0aadeeeb-2e52-442c-8837-85eda8a01852","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:44.955684892Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1835} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:44.955693052Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1835,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:44.956033982Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1835,"task":"crutest"} 01/30 Write block 1 len 16384 data: 5 6 10 10 02/30 Read block 536 len 4096 03/30 Read block 488 len 24576 04/30 Read block 409 len 4096 05/30 Read block 324 len 40960 06/30 Read block 146 len 32768 07/30 Read block 200 len 16384 08/30 Read block 204 len 32768 09/30 Flush 10/30 Write block 330 len 28672 data: 12 15 15 13 13 13 13 11/30 Flush 12/30 Write block 303 len 36864 data: 14 15 14 13 12 9 9 10 8 13/30 Write block 46 len 16384 data: 7 9 10 10 14/30 Write block 369 len 4096 data: 11 15/30 Write block 226 len 8192 data: 11 11 16/30 Write block 24 len 8192 data: 17 16 17/30 Read block 91 len 4096 18/30 Write block 215 len 32768 data: 16 17 16 16 16 12 11 11 19/30 Read block 263 len 24576 20/30 Write block 258 len 4096 data: 13 21/30 Read block 142 len 8192 22/30 Write block 270 len 16384 data: 19 21 20 14 23/30 Read block 141 len 32768 24/30 Write block 457 len 16384 data: 12 14 13 15 25/30 Read block 54 len 28672 26/30 Read block 318 len 28672 27/30 Read block 35 len 24576 28/30 Write block 302 len 24576 data: 17 15 16 15 14 13 29/30 Write block 530 len 40960 data: 11 11 11 12 10 10 9 10 11 11 30/30 Read block 255 len 8192 ---------------------------------------------------------------- Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Read 1 Sent Done Done false 1002 Acked Read 6 Sent Done Done false 1003 Acked Read 1 Sent Done Done false 1004 Acked Read 10 Sent Done Done false 1005 Acked Read 8 Sent Done Done false 1006 Acked Read 4 Sent Done Done false 1007 Acked Read 8 Sent Done Done false 1008 Acked Flush 0 Sent Done Done false 1009 Acked Write 7 Sent Done Done false 1010 Acked Flush 0 Sent Done Done false 1011 Acked Write 9 Sent Done Done false 1012 Acked Write 4 Sent Done Done false 1013 Acked Write 1 Sent Done Done false 1014 Acked Write 2 Sent Done Done false 1015 Acked Write 2 Sent Done Done false 1016 Acked Read 1 Sent Done Done false 1017 Acked Write 8 Sent Done Done false 1018 Acked Read 6 Sent Done Done false 1019 Acked Write 1 Sent Done Done false 1020 Acked Read 2 Sent Done Done false 1021 Acked Write 4 Sent Done Done false 1022 Acked Read 8 Sent Done Done false 1023 Acked Write 4 Sent Done Done false 1024 Acked Read 7 Sent Done Done false 1025 Acked Read 7 Sent Done Done false 1026 Acked Read 6 Sent Done Done false 1027 Acked Write 6 Sent Done Done false 1028 Acked Write 10 Sent Done Done false 1029 Acked Read 2 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 0 0 29 Done 1 30 30 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1010)) Some(JobId(1010)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:25:44.964399255Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1835} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:44.964593105Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1835,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 173 174 174 296 297 297 T F F <--- 1 020-039 173 173 173 296 296 296 F T T <--- 2 040-059 171 171 171 292 292 292 F T T <--- 3 060-079 172 172 172 293 293 293 F F F 4 080-099 166 166 166 280 280 280 F F F 5 100-119 172 172 172 294 294 294 F F F 6 120-139 173 173 173 296 296 296 F F F 7 140-159 168 168 168 283 283 283 F F F 8 160-179 173 173 173 296 296 296 F F F 9 180-199 173 173 173 296 296 296 F F F 10 200-219 172 172 172 295 295 295 F T T <--- 11 220-239 173 173 173 296 296 296 F T T <--- 12 240-259 169 169 169 286 286 286 F T T <--- 13 260-279 173 173 173 296 296 296 F T T <--- 14 280-299 173 173 173 296 296 296 F F F 15 300-319 166 166 166 280 280 280 F T T <--- 16 320-339 169 174 174 286 298 298 F F F <--- 17 340-359 168 168 168 285 285 285 F F F 18 360-379 164 164 164 276 276 276 F T T <--- 19 380-399 170 170 170 288 288 288 F F F 20 400-419 172 172 172 294 294 294 F F F 21 420-439 173 173 173 296 296 296 F F F 22 440-459 170 170 170 287 287 287 F T T <--- 23 460-479 168 168 168 284 284 284 F T T <--- 24 480-499 172 172 172 293 293 293 F F F 25 500-519 170 170 170 289 289 289 F F F 26 520-539 167 167 167 282 282 282 F T T <--- 27 540-559 163 163 163 274 274 274 F F F 28 560-579 172 172 172 294 294 294 F F F 29 580-599 172 172 172 294 294 294 F F F Max gen: 174, Max flush: 298 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 87 Verifying data now No change after verify Loop: 87 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 174 174 174 297 297 297 F F F 1 020-039 175 175 175 299 299 299 F F F 2 040-059 175 175 175 299 299 299 F F F 3 060-079 172 172 172 293 293 293 F F F 4 080-099 166 166 166 280 280 280 F F F 5 100-119 172 172 172 294 294 294 F F F 6 120-139 173 173 173 296 296 296 F F F 7 140-159 168 168 168 283 283 283 F F F 8 160-179 173 173 173 296 296 296 F F F 9 180-199 173 173 173 296 296 296 F F F 10 200-219 175 175 175 299 299 299 F F F 11 220-239 175 175 175 299 299 299 F F F 12 240-259 175 175 175 299 299 299 F F F 13 260-279 175 175 175 299 299 299 F F F 14 280-299 173 173 173 296 296 296 F F F 15 300-319 175 175 175 299 299 299 F F F 16 320-339 174 174 174 298 298 298 F F F 17 340-359 168 168 168 285 285 285 F F F 18 360-379 175 175 175 299 299 299 F F F 19 380-399 170 170 170 288 288 288 F F F 20 400-419 172 172 172 294 294 294 F F F 21 420-439 173 173 173 296 296 296 F F F 22 440-459 175 175 175 299 299 299 F F F 23 460-479 175 175 175 299 299 299 F F F 24 480-499 172 172 172 293 293 293 F F F 25 500-519 170 170 170 289 289 289 F F F 26 520-539 175 175 175 299 299 299 F F F 27 540-559 163 163 163 274 274 274 F F F 28 560-579 172 172 172 294 294 294 F F F 29 580-599 172 172 172 294 294 294 F F F Max gen: 175, Max flush: 299 Begin loop 88 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:49.223235832Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1843,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:49.231534744Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1843,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:49.231833895Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1843} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:49.231860925Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1843} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:49.231869625Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1843} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:49.231881425Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1843} {"msg":"Crucible 565fb316-2ab3-4c88-93c7-f815c3065809 has session id: 8809e21e-bd68-422b-8606-42f3d829bcdc","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:49.232278985Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1843,"session_id":"8809e21e-bd68-422b-8606-42f3d829bcdc"} {"msg":"Upstairs opts: Upstairs UUID: 565fb316-2ab3-4c88-93c7-f815c3065809, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:49.232309435Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1843,"session_id":"8809e21e-bd68-422b-8606-42f3d829bcdc"} {"msg":"Crucible stats registered with UUID: 565fb316-2ab3-4c88-93c7-f815c3065809","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:49.232322525Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1843,"session_id":"8809e21e-bd68-422b-8606-42f3d829bcdc"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:25:49.232331935Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1843,"session_id":"8809e21e-bd68-422b-8606-42f3d829bcdc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:49.232449666Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1843,"":"io task","client":"0","":"downstairs","session_id":"8809e21e-bd68-422b-8606-42f3d829bcdc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:49.232467646Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1843,"":"io task","client":"2","":"downstairs","session_id":"8809e21e-bd68-422b-8606-42f3d829bcdc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:49.232477736Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1843,"":"io task","client":"1","":"downstairs","session_id":"8809e21e-bd68-422b-8606-42f3d829bcdc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:49.232650666Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1843,"task":"crutest"} {"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:49.232668096Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1843} {"msg":"565fb316-2ab3-4c88-93c7-f815c3065809 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:49.232691706Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1843,"session_id":"8809e21e-bd68-422b-8606-42f3d829bcdc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:49.232701146Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1843,"client":"0","":"downstairs","session_id":"8809e21e-bd68-422b-8606-42f3d829bcdc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:49.232710926Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1843,"client":"1","":"downstairs","session_id":"8809e21e-bd68-422b-8606-42f3d829bcdc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:49.232720566Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1843,"client":"2","":"downstairs","session_id":"8809e21e-bd68-422b-8606-42f3d829bcdc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:49.232730166Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1843,"":"io task","client":"2","":"downstairs","session_id":"8809e21e-bd68-422b-8606-42f3d829bcdc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:49.232740236Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1843,"":"io task","client":"0","":"downstairs","session_id":"8809e21e-bd68-422b-8606-42f3d829bcdc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:49.232750306Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1843,"":"io task","client":"1","":"downstairs","session_id":"8809e21e-bd68-422b-8606-42f3d829bcdc"} {"msg":"ds_connection connected from Ok(127.0.0.1:63485)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:49.232786986Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1843,"":"io task","client":"2","":"downstairs","session_id":"8809e21e-bd68-422b-8606-42f3d829bcdc"} {"msg":"ds_connection connected from Ok(127.0.0.1:56913)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:49.232806736Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1843,"":"io task","client":"1","":"downstairs","session_id":"8809e21e-bd68-422b-8606-42f3d829bcdc"} {"msg":"ds_connection connected from Ok(127.0.0.1:44016)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:49.232821946Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1843,"":"io task","client":"0","":"downstairs","session_id":"8809e21e-bd68-422b-8606-42f3d829bcdc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:49.233471897Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1843,"client":"1","":"downstairs","session_id":"8809e21e-bd68-422b-8606-42f3d829bcdc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:49.233493767Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1843,"client":"2","":"downstairs","session_id":"8809e21e-bd68-422b-8606-42f3d829bcdc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:49.233510127Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1843,"client":"0","":"downstairs","session_id":"8809e21e-bd68-422b-8606-42f3d829bcdc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:49.233788208Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1843,"session_id":"8809e21e-bd68-422b-8606-42f3d829bcdc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:49.233812068Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1843,"session_id":"8809e21e-bd68-422b-8606-42f3d829bcdc"} {"msg":"[0]R flush_numbers[0..12]: [297, 299, 299, 293, 280, 294, 296, 283, 296, 296, 299, 299]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:49.233860558Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1843,"":"downstairs","":"downstairs","session_id":"8809e21e-bd68-422b-8606-42f3d829bcdc"} {"msg":"[0]R generation[0..12]: [174, 175, 175, 172, 166, 172, 173, 168, 173, 173, 175, 175]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:49.233872738Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1843,"":"downstairs","":"downstairs","session_id":"8809e21e-bd68-422b-8606-42f3d829bcdc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:49.233884158Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1843,"":"downstairs","":"downstairs","session_id":"8809e21e-bd68-422b-8606-42f3d829bcdc"} {"msg":"[1]R flush_numbers[0..12]: [297, 299, 299, 293, 280, 294, 296, 283, 296, 296, 299, 299]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:49.233894948Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1843,"":"downstairs","":"downstairs","session_id":"8809e21e-bd68-422b-8606-42f3d829bcdc"} {"msg":"[1]R generation[0..12]: [174, 175, 175, 172, 166, 172, 173, 168, 173, 173, 175, 175]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:49.233905718Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1843,"":"downstairs","":"downstairs","session_id":"8809e21e-bd68-422b-8606-42f3d829bcdc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:49.233916898Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1843,"":"downstairs","":"downstairs","session_id":"8809e21e-bd68-422b-8606-42f3d829bcdc"} {"msg":"[2]R flush_numbers[0..12]: [297, 299, 299, 293, 280, 294, 296, 283, 296, 296, 299, 299]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:49.233927848Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1843,"":"downstairs","":"downstairs","session_id":"8809e21e-bd68-422b-8606-42f3d829bcdc"} {"msg":"[2]R generation[0..12]: [174, 175, 175, 172, 166, 172, 173, 168, 173, 173, 175, 175]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:49.233938658Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1843,"":"downstairs","":"downstairs","session_id":"8809e21e-bd68-422b-8606-42f3d829bcdc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:49.233951728Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1843,"":"downstairs","":"downstairs","session_id":"8809e21e-bd68-422b-8606-42f3d829bcdc"} {"msg":"Max found gen is 176","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:49.233961418Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1843,"":"downstairs","":"downstairs","session_id":"8809e21e-bd68-422b-8606-42f3d829bcdc"} {"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:49.233971458Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1843,"":"downstairs","":"downstairs","session_id":"8809e21e-bd68-422b-8606-42f3d829bcdc"} {"msg":"Next flush: 300","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:49.233980968Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1843,"":"downstairs","":"downstairs","session_id":"8809e21e-bd68-422b-8606-42f3d829bcdc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:49.233990568Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1843,"":"downstairs","":"downstairs","session_id":"8809e21e-bd68-422b-8606-42f3d829bcdc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:49.234012568Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1843,"session_id":"8809e21e-bd68-422b-8606-42f3d829bcdc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:49.234021468Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1843,"client":"0","":"downstairs","session_id":"8809e21e-bd68-422b-8606-42f3d829bcdc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:49.234031418Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1843,"client":"1","":"downstairs","session_id":"8809e21e-bd68-422b-8606-42f3d829bcdc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:49.234041258Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1843,"client":"2","":"downstairs","session_id":"8809e21e-bd68-422b-8606-42f3d829bcdc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:49.234051208Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1843,"session_id":"8809e21e-bd68-422b-8606-42f3d829bcdc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:49.234060638Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1843,"session_id":"8809e21e-bd68-422b-8606-42f3d829bcdc"} {"msg":"565fb316-2ab3-4c88-93c7-f815c3065809 is now active with session: 8809e21e-bd68-422b-8606-42f3d829bcdc","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:49.234070868Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1843,"session_id":"8809e21e-bd68-422b-8606-42f3d829bcdc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:49.234079798Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1843,"session_id":"8809e21e-bd68-422b-8606-42f3d829bcdc"} {"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:49.234089138Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1843} {"msg":"Activated sub_volume 565fb316-2ab3-4c88-93c7-f815c3065809","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:49.234097368Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1843} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:49.234105548Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1843,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:49.234420929Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1843,"task":"crutest"} 01/30 Write block 274 len 8192 data: 16 18 02/30 Flush 03/30 Write block 433 len 12288 data: 9 8 7 04/30 Read block 243 len 20480 05/30 Read block 342 len 8192 06/30 Write block 542 len 36864 data: 12 16 19 15 12 11 12 12 12 07/30 Read block 300 len 16384 08/30 Read block 61 len 12288 09/30 Read block 94 len 36864 10/30 Write block 282 len 28672 data: 13 12 10 12 12 11 11 11/30 Read block 276 len 12288 12/30 Write block 459 len 28672 data: 14 16 15 16 20 18 16 13/30 Write block 39 len 36864 data: 10 8 10 10 10 6 6 8 10 14/30 Write block 396 len 4096 data: 9 15/30 Write block 162 len 20480 data: 8 7 8 8 9 16/30 Read block 92 len 24576 17/30 Read block 13 len 32768 18/30 Write block 108 len 12288 data: 12 13 14 19/30 Read block 286 len 20480 20/30 Flush 21/30 Write block 393 len 16384 data: 10 11 11 10 22/30 Read block 262 len 20480 23/30 Read block 434 len 12288 24/30 Write block 85 len 40960 data: 17 16 15 16 18 18 18 17 15 15 25/30 Write block 29 len 32768 data: 14 16 16 13 11 9 7 8 26/30 Write block 258 len 28672 data: 14 12 14 16 15 17 16 27/30 Write block 57 len 36864 data: 11 13 13 13 16 16 16 14 13 28/30 Write block 339 len 28672 data: 15 16 17 13 13 14 13 29/30 Write block 316 len 4096 data: 10 30/30 Read block 30 len 24576 ---------------------------------------------------------------- Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Sent false 1001 Acked Flush 0 Done Done Sent false 1002 Acked Write 3 Done Done Sent false 1003 Acked Read 5 Done Done Sent false 1004 Acked Read 2 Done Done Sent false 1005 Acked Write 9 Done Done Sent false 1006 Acked Read 4 Done Done Sent false 1007 Acked Read 3 Done Done Sent false 1008 Acked Read 9 Done Done Sent false 1009 Acked Write 7 Done Done Sent false 1010 Acked Read 3 Done Done Sent false 1011 Acked Write 7 Done Done Sent false 1012 Acked Write 9 Done Done Sent false 1013 Acked Write 1 Done Done Sent false 1014 Acked Write 5 Done Done Sent false 1015 Acked Read 6 Done Done Sent false 1016 Acked Read 8 Done Done Sent false 1017 Acked Write 3 Done Done Sent false 1018 Acked Read 5 Done Done Sent false 1019 Acked Flush 0 Done Done Sent false 1020 Acked Write 4 Done Done Sent false 1021 Acked Read 5 Done Done Sent false 1022 Acked Read 3 Done Done Sent false 1023 Acked Write 10 Done Done Sent false 1024 Acked Write 8 Done Done Sent false 1025 Acked Write 7 Done Done Sent false 1026 Acked Write 9 Done Done Sent false 1027 Acked Write 7 Done Done Sent false 1028 Acked Write 1 Done Done Sent false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 30 31 Done 30 29 0 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:25:49.243080572Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1843} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:49.243265362Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1843,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 174 174 174 297 297 297 F F F 1 020-039 176 176 175 301 301 299 T T F <--- 2 040-059 176 176 175 301 301 299 T T F <--- 3 060-079 172 172 172 293 293 293 T T F <--- 4 080-099 166 166 166 280 280 280 T T F <--- 5 100-119 176 176 172 301 301 294 F F F <--- 6 120-139 173 173 173 296 296 296 F F F 7 140-159 168 168 168 283 283 283 F F F 8 160-179 176 176 173 301 301 296 F F F <--- 9 180-199 173 173 173 296 296 296 F F F 10 200-219 175 175 175 299 299 299 F F F 11 220-239 175 175 175 299 299 299 F F F 12 240-259 175 175 175 299 299 299 T T F <--- 13 260-279 176 176 175 300 300 299 T T F <--- 14 280-299 176 176 173 301 301 296 F F F <--- 15 300-319 175 175 175 299 299 299 T T F <--- 16 320-339 174 174 174 298 298 298 T T F <--- 17 340-359 168 168 168 285 285 285 T T F <--- 18 360-379 175 175 175 299 299 299 F F F 19 380-399 176 176 170 301 301 288 T T F <--- 20 400-419 172 172 172 294 294 294 F F F 21 420-439 176 176 173 301 301 296 F F F <--- 22 440-459 176 176 175 301 301 299 F F F <--- 23 460-479 176 176 175 301 301 299 F F F <--- 24 480-499 172 172 172 293 293 293 F F F 25 500-519 170 170 170 289 289 289 F F F 26 520-539 175 175 175 299 299 299 F F F 27 540-559 176 176 163 301 301 274 F F F <--- 28 560-579 172 172 172 294 294 294 F F F 29 580-599 172 172 172 294 294 294 F F F Max gen: 176, Max flush: 301 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 88 Verifying data now No change after verify Loop: 88 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 174 174 174 297 297 297 F F F 1 020-039 177 177 177 302 302 302 F F F 2 040-059 177 177 177 302 302 302 F F F 3 060-079 177 177 177 302 302 302 F F F 4 080-099 177 177 177 302 302 302 F F F 5 100-119 176 176 176 301 301 301 F F F 6 120-139 173 173 173 296 296 296 F F F 7 140-159 168 168 168 283 283 283 F F F 8 160-179 176 176 176 301 301 301 F F F 9 180-199 173 173 173 296 296 296 F F F 10 200-219 175 175 175 299 299 299 F F F 11 220-239 175 175 175 299 299 299 F F F 12 240-259 177 177 177 302 302 302 F F F 13 260-279 177 177 177 302 302 302 F F F 14 280-299 176 176 176 301 301 301 F F F 15 300-319 177 177 177 302 302 302 F F F 16 320-339 177 177 177 302 302 302 F F F 17 340-359 177 177 177 302 302 302 F F F 18 360-379 175 175 175 299 299 299 F F F 19 380-399 177 177 177 302 302 302 F F F 20 400-419 172 172 172 294 294 294 F F F 21 420-439 176 176 176 301 301 301 F F F 22 440-459 176 176 176 301 301 301 F F F 23 460-479 176 176 176 301 301 301 F F F 24 480-499 172 172 172 293 293 293 F F F 25 500-519 170 170 170 289 289 289 F F F 26 520-539 175 175 175 299 299 299 F F F 27 540-559 176 176 176 301 301 301 F F F 28 560-579 172 172 172 294 294 294 F F F 29 580-599 172 172 172 294 294 294 F F F Max gen: 177, Max flush: 302 Begin loop 89 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:53.504565572Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1851,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:25:53.511708283Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1851,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:53.511760173Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1851,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:53.518734034Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1851,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:53.519001694Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1851} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:53.519032294Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1851} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:53.519041044Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1851} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:53.519052854Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1851} {"msg":"Crucible dc4f67b8-3ee3-4236-b201-6a40d7956d4d has session id: de952517-95ad-492a-9af4-104f7c670f92","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:53.519381505Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1851,"session_id":"de952517-95ad-492a-9af4-104f7c670f92"} {"msg":"Upstairs opts: Upstairs UUID: dc4f67b8-3ee3-4236-b201-6a40d7956d4d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:53.519406745Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1851,"session_id":"de952517-95ad-492a-9af4-104f7c670f92"} {"msg":"Crucible stats registered with UUID: dc4f67b8-3ee3-4236-b201-6a40d7956d4d","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:53.519416945Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1851,"session_id":"de952517-95ad-492a-9af4-104f7c670f92"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:25:53.519426095Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1851,"session_id":"de952517-95ad-492a-9af4-104f7c670f92"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:53.519556275Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1851,"":"io task","client":"1","":"downstairs","session_id":"de952517-95ad-492a-9af4-104f7c670f92"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:53.519576365Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1851,"":"io task","client":"0","":"downstairs","session_id":"de952517-95ad-492a-9af4-104f7c670f92"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:53.519586565Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1851,"":"io task","client":"2","":"downstairs","session_id":"de952517-95ad-492a-9af4-104f7c670f92"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:53.519743035Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1851,"task":"crutest"} {"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:53.519767045Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1851} {"msg":"dc4f67b8-3ee3-4236-b201-6a40d7956d4d active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:53.519791005Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1851,"session_id":"de952517-95ad-492a-9af4-104f7c670f92"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:53.519800465Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1851,"client":"0","":"downstairs","session_id":"de952517-95ad-492a-9af4-104f7c670f92"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:53.519810195Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1851,"client":"1","":"downstairs","session_id":"de952517-95ad-492a-9af4-104f7c670f92"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:53.519820065Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1851,"client":"2","":"downstairs","session_id":"de952517-95ad-492a-9af4-104f7c670f92"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:53.519829615Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1851,"":"io task","client":"2","":"downstairs","session_id":"de952517-95ad-492a-9af4-104f7c670f92"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:53.519839695Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1851,"":"io task","client":"0","":"downstairs","session_id":"de952517-95ad-492a-9af4-104f7c670f92"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:53.519849715Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1851,"":"io task","client":"1","":"downstairs","session_id":"de952517-95ad-492a-9af4-104f7c670f92"} {"msg":"ds_connection connected from Ok(127.0.0.1:59815)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:53.519893635Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1851,"":"io task","client":"1","":"downstairs","session_id":"de952517-95ad-492a-9af4-104f7c670f92"} {"msg":"ds_connection connected from Ok(127.0.0.1:63435)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:53.519907236Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1851,"":"io task","client":"2","":"downstairs","session_id":"de952517-95ad-492a-9af4-104f7c670f92"} {"msg":"ds_connection connected from Ok(127.0.0.1:56608)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:53.520031796Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1851,"":"io task","client":"0","":"downstairs","session_id":"de952517-95ad-492a-9af4-104f7c670f92"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:53.520604557Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1851,"client":"2","":"downstairs","session_id":"de952517-95ad-492a-9af4-104f7c670f92"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:53.520630307Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1851,"client":"1","":"downstairs","session_id":"de952517-95ad-492a-9af4-104f7c670f92"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:53.520645157Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1851,"client":"0","":"downstairs","session_id":"de952517-95ad-492a-9af4-104f7c670f92"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:53.520924907Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1851,"session_id":"de952517-95ad-492a-9af4-104f7c670f92"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:53.520936797Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1851,"session_id":"de952517-95ad-492a-9af4-104f7c670f92"} {"msg":"[0]R flush_numbers[0..12]: [297, 302, 302, 302, 302, 301, 296, 283, 301, 296, 299, 299]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:53.520969097Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1851,"":"downstairs","":"downstairs","session_id":"de952517-95ad-492a-9af4-104f7c670f92"} {"msg":"[0]R generation[0..12]: [174, 177, 177, 177, 177, 176, 173, 168, 176, 173, 175, 175]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:53.520980357Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1851,"":"downstairs","":"downstairs","session_id":"de952517-95ad-492a-9af4-104f7c670f92"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:53.520991647Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1851,"":"downstairs","":"downstairs","session_id":"de952517-95ad-492a-9af4-104f7c670f92"} {"msg":"[1]R flush_numbers[0..12]: [297, 302, 302, 302, 302, 301, 296, 283, 301, 296, 299, 299]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:53.521012787Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1851,"":"downstairs","":"downstairs","session_id":"de952517-95ad-492a-9af4-104f7c670f92"} {"msg":"[1]R generation[0..12]: [174, 177, 177, 177, 177, 176, 173, 168, 176, 173, 175, 175]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:53.521023517Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1851,"":"downstairs","":"downstairs","session_id":"de952517-95ad-492a-9af4-104f7c670f92"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:53.521034627Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1851,"":"downstairs","":"downstairs","session_id":"de952517-95ad-492a-9af4-104f7c670f92"} {"msg":"[2]R flush_numbers[0..12]: [297, 302, 302, 302, 302, 301, 296, 283, 301, 296, 299, 299]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:53.521045287Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1851,"":"downstairs","":"downstairs","session_id":"de952517-95ad-492a-9af4-104f7c670f92"} {"msg":"[2]R generation[0..12]: [174, 177, 177, 177, 177, 176, 173, 168, 176, 173, 175, 175]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:53.521055837Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1851,"":"downstairs","":"downstairs","session_id":"de952517-95ad-492a-9af4-104f7c670f92"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:53.521066857Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1851,"":"downstairs","":"downstairs","session_id":"de952517-95ad-492a-9af4-104f7c670f92"} {"msg":"Max found gen is 178","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:53.521076417Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1851,"":"downstairs","":"downstairs","session_id":"de952517-95ad-492a-9af4-104f7c670f92"} {"msg":"Generation requested: 178 >= found:178","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:53.521086357Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1851,"":"downstairs","":"downstairs","session_id":"de952517-95ad-492a-9af4-104f7c670f92"} {"msg":"Next flush: 303","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:53.521097677Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1851,"":"downstairs","":"downstairs","session_id":"de952517-95ad-492a-9af4-104f7c670f92"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:53.521107127Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1851,"":"downstairs","":"downstairs","session_id":"de952517-95ad-492a-9af4-104f7c670f92"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:53.521117327Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1851,"session_id":"de952517-95ad-492a-9af4-104f7c670f92"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:53.521126267Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1851,"client":"0","":"downstairs","session_id":"de952517-95ad-492a-9af4-104f7c670f92"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:53.521136397Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1851,"client":"1","":"downstairs","session_id":"de952517-95ad-492a-9af4-104f7c670f92"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:53.521146237Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1851,"client":"2","":"downstairs","session_id":"de952517-95ad-492a-9af4-104f7c670f92"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:53.521156317Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1851,"session_id":"de952517-95ad-492a-9af4-104f7c670f92"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:53.521165777Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1851,"session_id":"de952517-95ad-492a-9af4-104f7c670f92"} {"msg":"dc4f67b8-3ee3-4236-b201-6a40d7956d4d is now active with session: de952517-95ad-492a-9af4-104f7c670f92","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:53.521175917Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1851,"session_id":"de952517-95ad-492a-9af4-104f7c670f92"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:53.521184797Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1851,"session_id":"de952517-95ad-492a-9af4-104f7c670f92"} {"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:53.521193997Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1851} {"msg":"Activated sub_volume dc4f67b8-3ee3-4236-b201-6a40d7956d4d","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:53.521202007Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1851} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-07-22T22:25:53.521210117Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1851,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:53.521456188Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1851,"task":"crutest"} 01/30 Write block 407 len 8192 data: 12 13 02/30 Write block 155 len 28672 data: 14 15 15 18 15 14 13 03/30 Read block 112 len 20480 04/30 Write block 581 len 32768 data: 12 13 15 17 14 15 17 16 05/30 Read block 289 len 40960 06/30 Read block 122 len 4096 07/30 Write block 299 len 36864 data: 18 16 17 18 16 17 16 15 14 08/30 Write block 363 len 4096 data: 17 09/30 Write block 197 len 20480 data: 13 9 8 10 11 10/30 Flush 11/30 Write block 374 len 32768 data: 13 14 16 16 15 17 16 17 12/30 Read block 94 len 4096 13/30 Write block 285 len 16384 data: 13 13 12 12 14/30 Write block 584 len 16384 data: 18 15 16 18 15/30 Write block 171 len 20480 data: 15 16 16 13 11 16/30 Read block 134 len 12288 17/30 Read block 333 len 4096 18/30 Read block 256 len 8192 19/30 Read block 335 len 12288 20/30 Flush 21/30 Write block 249 len 28672 data: 15 14 17 12 10 9 11 22/30 Write block 262 len 20480 data: 16 18 17 17 16 23/30 Flush 24/30 Write block 222 len 28672 data: 12 13 10 12 12 12 12 25/30 Write block 115 len 8192 data: 17 16 26/30 Read block 296 len 32768 27/30 Write block 394 len 16384 data: 12 12 11 9 28/30 Read block 471 len 16384 29/30 Read block 219 len 12288 30/30 Write block 449 len 20480 data: 9 8 8 9 9 ---------------------------------------------------------------- Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Write 7 Done Done Done false 1002 Acked Read 5 Sent Done Done false 1003 Acked Write 8 Sent Done Done false 1004 Acked Read 10 Sent Done Done false 1005 Acked Read 1 Sent Done Done false 1006 Acked Write 9 Sent Done Done false 1007 Acked Write 1 Sent Done Done false 1008 Acked Write 5 Sent Done Done false 1009 Acked Flush 0 Sent Done Done false 1010 Acked Write 8 Sent Done Done false 1011 Acked Read 1 Sent Done Done false 1012 Acked Write 4 Sent Done Done false 1013 Acked Write 4 Sent Done Done false 1014 Acked Write 5 Sent Done Done false 1015 Acked Read 3 Sent Done Done false 1016 Acked Read 1 Sent Done Done false 1017 Acked Read 2 Sent Done Done false 1018 Acked Read 3 Sent Done Done false 1019 Acked Flush 0 Sent Done Done false 1020 Acked Write 7 Sent Done Done false 1021 Acked Write 5 Sent Done Done false 1022 Acked Flush 0 Sent Done Done false 1023 Acked Write 7 Sent Done Done false 1024 Acked Write 2 Sent Done Done false 1025 Acked Read 8 Sent Done Done false 1026 Acked Write 4 Sent Done Done false 1027 Acked Read 4 Sent Done Done false 1028 Acked Read 3 Sent Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 1 1 30 Done 2 29 29 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:25:53.529886221Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1851} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:53.530066651Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1851,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 174 174 174 297 297 297 F F F 1 020-039 177 177 177 302 302 302 F F F 2 040-059 177 177 177 302 302 302 F F F 3 060-079 177 177 177 302 302 302 F F F 4 080-099 177 177 177 302 302 302 F F F 5 100-119 176 176 176 301 301 301 F T T <--- 6 120-139 173 173 173 296 296 296 F F F 7 140-159 168 178 178 283 303 303 T F F <--- 8 160-179 176 178 178 301 304 304 T F F <--- 9 180-199 173 178 178 296 303 303 F F F <--- 10 200-219 175 178 178 299 303 303 F F F <--- 11 220-239 175 175 175 299 299 299 F T T <--- 12 240-259 177 178 178 302 305 305 F F F <--- 13 260-279 177 178 178 302 305 305 F F F <--- 14 280-299 176 178 178 301 304 304 F F F <--- 15 300-319 177 178 178 302 303 303 F F F <--- 16 320-339 177 177 177 302 302 302 F F F 17 340-359 177 177 177 302 302 302 F F F 18 360-379 175 178 178 299 304 304 F F F <--- 19 380-399 177 178 178 302 304 304 F T T <--- 20 400-419 172 178 178 294 303 303 T F F <--- 21 420-439 176 176 176 301 301 301 F F F 22 440-459 176 176 176 301 301 301 F T T <--- 23 460-479 176 176 176 301 301 301 F F F 24 480-499 172 172 172 293 293 293 F F F 25 500-519 170 170 170 289 289 289 F F F 26 520-539 175 175 175 299 299 299 F F F 27 540-559 176 176 176 301 301 301 F F F 28 560-579 172 172 172 294 294 294 F F F 29 580-599 172 178 178 294 304 304 F F F <--- Max gen: 178, Max flush: 305 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 89 Verifying data now No change after verify Loop: 89 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 174 174 174 297 297 297 F F F 1 020-039 177 177 177 302 302 302 F F F 2 040-059 177 177 177 302 302 302 F F F 3 060-079 177 177 177 302 302 302 F F F 4 080-099 177 177 177 302 302 302 F F F 5 100-119 179 179 179 306 306 306 F F F 6 120-139 173 173 173 296 296 296 F F F 7 140-159 178 178 178 303 303 303 F F F 8 160-179 178 178 178 304 304 304 F F F 9 180-199 178 178 178 303 303 303 F F F 10 200-219 178 178 178 303 303 303 F F F 11 220-239 179 179 179 306 306 306 F F F 12 240-259 178 178 178 305 305 305 F F F 13 260-279 178 178 178 305 305 305 F F F 14 280-299 178 178 178 304 304 304 F F F 15 300-319 178 178 178 303 303 303 F F F 16 320-339 177 177 177 302 302 302 F F F 17 340-359 177 177 177 302 302 302 F F F 18 360-379 178 178 178 304 304 304 F F F 19 380-399 179 179 179 306 306 306 F F F 20 400-419 178 178 178 303 303 303 F F F 21 420-439 176 176 176 301 301 301 F F F 22 440-459 179 179 179 306 306 306 F F F 23 460-479 176 176 176 301 301 301 F F F 24 480-499 172 172 172 293 293 293 F F F 25 500-519 170 170 170 289 289 289 F F F 26 520-539 175 175 175 299 299 299 F F F 27 540-559 176 176 176 301 301 301 F F F 28 560-579 172 172 172 294 294 294 F F F 29 580-599 178 178 178 304 304 304 F F F Max gen: 179, Max flush: 306 Begin loop 90 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:57.79028349Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1859,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:57.798547382Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1859,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:57.798842013Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1859} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:57.798869103Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1859} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:57.798877793Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1859} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:57.798889633Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1859} {"msg":"Crucible fb089faa-000d-4e61-acbe-52d731410172 has session id: d5b31a9e-8ab7-4576-9812-6d228bce0152","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:57.799267563Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1859,"session_id":"d5b31a9e-8ab7-4576-9812-6d228bce0152"} {"msg":"Upstairs opts: Upstairs UUID: fb089faa-000d-4e61-acbe-52d731410172, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:57.799296733Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1859,"session_id":"d5b31a9e-8ab7-4576-9812-6d228bce0152"} {"msg":"Crucible stats registered with UUID: fb089faa-000d-4e61-acbe-52d731410172","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:57.799307523Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1859,"session_id":"d5b31a9e-8ab7-4576-9812-6d228bce0152"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:25:57.799316533Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1859,"session_id":"d5b31a9e-8ab7-4576-9812-6d228bce0152"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:57.799433273Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1859,"":"io task","client":"0","":"downstairs","session_id":"d5b31a9e-8ab7-4576-9812-6d228bce0152"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:57.799451053Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1859,"":"io task","client":"1","":"downstairs","session_id":"d5b31a9e-8ab7-4576-9812-6d228bce0152"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:57.799461313Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1859,"":"io task","client":"2","":"downstairs","session_id":"d5b31a9e-8ab7-4576-9812-6d228bce0152"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:57.799624364Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1859,"task":"crutest"} {"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:57.799641174Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1859} {"msg":"fb089faa-000d-4e61-acbe-52d731410172 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:57.799657214Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1859,"session_id":"d5b31a9e-8ab7-4576-9812-6d228bce0152"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:57.799666094Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1859,"client":"0","":"downstairs","session_id":"d5b31a9e-8ab7-4576-9812-6d228bce0152"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:57.799677864Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1859,"client":"1","":"downstairs","session_id":"d5b31a9e-8ab7-4576-9812-6d228bce0152"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:57.799687734Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1859,"client":"2","":"downstairs","session_id":"d5b31a9e-8ab7-4576-9812-6d228bce0152"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:57.799697344Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1859,"":"io task","client":"2","":"downstairs","session_id":"d5b31a9e-8ab7-4576-9812-6d228bce0152"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:57.799707524Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1859,"":"io task","client":"0","":"downstairs","session_id":"d5b31a9e-8ab7-4576-9812-6d228bce0152"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:57.799717474Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1859,"":"io task","client":"1","":"downstairs","session_id":"d5b31a9e-8ab7-4576-9812-6d228bce0152"} {"msg":"ds_connection connected from Ok(127.0.0.1:58152)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:57.799860214Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1859,"":"io task","client":"1","":"downstairs","session_id":"d5b31a9e-8ab7-4576-9812-6d228bce0152"} {"msg":"ds_connection connected from Ok(127.0.0.1:47484)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:57.799871764Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1859,"":"io task","client":"0","":"downstairs","session_id":"d5b31a9e-8ab7-4576-9812-6d228bce0152"} {"msg":"ds_connection connected from Ok(127.0.0.1:39072)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:57.799882244Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1859,"":"io task","client":"2","":"downstairs","session_id":"d5b31a9e-8ab7-4576-9812-6d228bce0152"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:57.800471105Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1859,"client":"1","":"downstairs","session_id":"d5b31a9e-8ab7-4576-9812-6d228bce0152"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:57.800524705Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1859,"client":"0","":"downstairs","session_id":"d5b31a9e-8ab7-4576-9812-6d228bce0152"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:57.800589025Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1859,"client":"2","":"downstairs","session_id":"d5b31a9e-8ab7-4576-9812-6d228bce0152"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:57.800794216Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1859,"session_id":"d5b31a9e-8ab7-4576-9812-6d228bce0152"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:57.800811926Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1859,"session_id":"d5b31a9e-8ab7-4576-9812-6d228bce0152"} {"msg":"[0]R flush_numbers[0..12]: [297, 302, 302, 302, 302, 306, 296, 303, 304, 303, 303, 306]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:57.800867206Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1859,"":"downstairs","":"downstairs","session_id":"d5b31a9e-8ab7-4576-9812-6d228bce0152"} {"msg":"[0]R generation[0..12]: [174, 177, 177, 177, 177, 179, 173, 178, 178, 178, 178, 179]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:57.800878706Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1859,"":"downstairs","":"downstairs","session_id":"d5b31a9e-8ab7-4576-9812-6d228bce0152"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:57.800891926Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1859,"":"downstairs","":"downstairs","session_id":"d5b31a9e-8ab7-4576-9812-6d228bce0152"} {"msg":"[1]R flush_numbers[0..12]: [297, 302, 302, 302, 302, 306, 296, 303, 304, 303, 303, 306]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:57.800902796Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1859,"":"downstairs","":"downstairs","session_id":"d5b31a9e-8ab7-4576-9812-6d228bce0152"} {"msg":"[1]R generation[0..12]: [174, 177, 177, 177, 177, 179, 173, 178, 178, 178, 178, 179]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:57.800913516Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1859,"":"downstairs","":"downstairs","session_id":"d5b31a9e-8ab7-4576-9812-6d228bce0152"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:57.800924386Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1859,"":"downstairs","":"downstairs","session_id":"d5b31a9e-8ab7-4576-9812-6d228bce0152"} {"msg":"[2]R flush_numbers[0..12]: [297, 302, 302, 302, 302, 306, 296, 303, 304, 303, 303, 306]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:57.800935106Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1859,"":"downstairs","":"downstairs","session_id":"d5b31a9e-8ab7-4576-9812-6d228bce0152"} {"msg":"[2]R generation[0..12]: [174, 177, 177, 177, 177, 179, 173, 178, 178, 178, 178, 179]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:57.800945756Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1859,"":"downstairs","":"downstairs","session_id":"d5b31a9e-8ab7-4576-9812-6d228bce0152"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:57.800956746Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1859,"":"downstairs","":"downstairs","session_id":"d5b31a9e-8ab7-4576-9812-6d228bce0152"} {"msg":"Max found gen is 180","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:57.800966196Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1859,"":"downstairs","":"downstairs","session_id":"d5b31a9e-8ab7-4576-9812-6d228bce0152"} {"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:57.800976006Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1859,"":"downstairs","":"downstairs","session_id":"d5b31a9e-8ab7-4576-9812-6d228bce0152"} {"msg":"Next flush: 307","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:57.800985466Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1859,"":"downstairs","":"downstairs","session_id":"d5b31a9e-8ab7-4576-9812-6d228bce0152"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:57.800995036Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1859,"":"downstairs","":"downstairs","session_id":"d5b31a9e-8ab7-4576-9812-6d228bce0152"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:57.801004966Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1859,"session_id":"d5b31a9e-8ab7-4576-9812-6d228bce0152"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:57.801013746Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1859,"client":"0","":"downstairs","session_id":"d5b31a9e-8ab7-4576-9812-6d228bce0152"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:57.801023536Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1859,"client":"1","":"downstairs","session_id":"d5b31a9e-8ab7-4576-9812-6d228bce0152"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:57.801034736Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1859,"client":"2","":"downstairs","session_id":"d5b31a9e-8ab7-4576-9812-6d228bce0152"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:57.801044626Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1859,"session_id":"d5b31a9e-8ab7-4576-9812-6d228bce0152"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:57.801053686Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1859,"session_id":"d5b31a9e-8ab7-4576-9812-6d228bce0152"} {"msg":"fb089faa-000d-4e61-acbe-52d731410172 is now active with session: d5b31a9e-8ab7-4576-9812-6d228bce0152","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:57.801063836Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1859,"session_id":"d5b31a9e-8ab7-4576-9812-6d228bce0152"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:57.801072676Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1859,"session_id":"d5b31a9e-8ab7-4576-9812-6d228bce0152"} {"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:57.801081936Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1859} {"msg":"Activated sub_volume fb089faa-000d-4e61-acbe-52d731410172","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:57.801089976Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1859} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:57.801098076Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1859,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:57.801422156Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1859,"task":"crutest"} 01/30 Write block 310 len 12288 data: 11 9 8 02/30 Read block 168 len 4096 03/30 Read block 102 len 32768 04/30 Flush 05/30 Write block 456 len 4096 data: 16 06/30 Write block 205 len 28672 data: 12 11 11 14 15 15 16 07/30 Read block 175 len 20480 08/30 Read block 405 len 36864 09/30 Read block 172 len 36864 10/30 Write block 155 len 4096 data: 15 11/30 Write block 58 len 32768 data: 14 14 14 17 17 17 15 14 12/30 Flush 13/30 Write block 439 len 40960 data: 11 9 8 10 10 9 8 8 9 9 14/30 Read block 374 len 28672 15/30 Read block 403 len 36864 16/30 Read block 478 len 20480 17/30 Read block 213 len 24576 18/30 Write block 294 len 8192 data: 17 17 19/30 Write block 478 len 32768 data: 13 9 10 9 9 11 13 15 20/30 Write block 75 len 40960 data: 8 10 12 16 17 17 13 13 17 16 21/30 Write block 59 len 28672 data: 15 15 18 18 18 16 15 22/30 Read block 296 len 40960 23/30 Read block 221 len 36864 24/30 Write block 420 len 8192 data: 14 14 25/30 Read block 239 len 32768 26/30 Read block 343 len 8192 27/30 Write block 133 len 12288 data: 10 9 12 28/30 Write block 430 len 40960 data: 9 10 7 10 9 8 7 10 11 12 29/30 Write block 445 len 12288 data: 9 9 10 30/30 Read block 238 len 36864 ---------------------------------------------------------------- Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Sent false 1001 Acked Read 1 Done Done Sent false 1002 Acked Read 8 Done Done Sent false 1003 Acked Flush 0 Done Done Sent false 1004 Acked Write 1 Done Done Sent false 1005 Acked Write 7 Done Done Sent false 1006 Acked Read 5 Done Done Sent false 1007 Acked Read 9 Done Done Sent false 1008 Acked Read 9 Done Done Sent false 1009 Acked Write 1 Done Done Sent false 1010 Acked Write 8 Done Done Sent false 1011 Acked Flush 0 Done Done Sent false 1012 Acked Write 10 Done Done Sent false 1013 Acked Read 7 Done Done Sent false 1014 Acked Read 9 Done Done Sent false 1015 Acked Read 5 Done Done Sent false 1016 Acked Read 6 Done Done Sent false 1017 Acked Write 2 Done Done Sent false 1018 Acked Write 8 Done Done Sent false 1019 Acked Write 10 Done Done Sent false 1020 Acked Write 7 Done Done Sent false 1021 Acked Read 10 Done Done Sent false 1022 Acked Read 9 Done Done Sent false 1023 Acked Write 2 Done Done Sent false 1024 Acked Read 8 Done Done Sent false 1025 Acked Read 2 Done Done Sent false 1026 Acked Write 3 Done Done Sent false 1027 Acked Write 10 Done Done Sent false 1028 Acked Write 3 Done Done Sent false 1029 Acked Read 9 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 30 31 Done 29 30 0 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1011)) Some(JobId(1011)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:25:57.81018918Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1859} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:25:57.81037495Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1859,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 174 174 174 297 297 297 F F F 1 020-039 177 177 177 302 302 302 F F F 2 040-059 180 180 177 308 308 302 T T F <--- 3 060-079 180 180 177 308 308 302 T T F <--- 4 080-099 177 177 177 302 302 302 T T F <--- 5 100-119 179 179 179 306 306 306 F F F 6 120-139 173 173 173 296 296 296 T T F <--- 7 140-159 180 180 178 308 308 303 F F F <--- 8 160-179 178 178 178 304 304 304 F F F 9 180-199 178 178 178 303 303 303 F F F 10 200-219 180 180 178 308 308 303 F F F <--- 11 220-239 179 179 179 306 306 306 F F F 12 240-259 178 178 178 305 305 305 F F F 13 260-279 178 178 178 305 305 305 F F F 14 280-299 178 178 178 304 304 304 T T F <--- 15 300-319 180 180 178 307 307 303 F F F <--- 16 320-339 177 177 177 302 302 302 F F F 17 340-359 177 177 177 302 302 302 F F F 18 360-379 178 178 178 304 304 304 F F F 19 380-399 179 179 179 306 306 306 F F F 20 400-419 178 178 178 303 303 303 F F F 21 420-439 176 176 176 301 301 301 T T F <--- 22 440-459 180 180 179 308 308 306 T T F <--- 23 460-479 176 176 176 301 301 301 T T F <--- 24 480-499 172 172 172 293 293 293 T T F <--- 25 500-519 170 170 170 289 289 289 F F F 26 520-539 175 175 175 299 299 299 F F F 27 540-559 176 176 176 301 301 301 F F F 28 560-579 172 172 172 294 294 294 F F F 29 580-599 178 178 178 304 304 304 F F F Max gen: 180, Max flush: 308 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 90 Verifying data now No change after verify Loop: 90 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 174 174 174 297 297 297 F F F 1 020-039 177 177 177 302 302 302 F F F 2 040-059 181 181 181 309 309 309 F F F 3 060-079 181 181 181 309 309 309 F F F 4 080-099 181 181 181 309 309 309 F F F 5 100-119 179 179 179 306 306 306 F F F 6 120-139 181 181 181 309 309 309 F F F 7 140-159 180 180 180 308 308 308 F F F 8 160-179 178 178 178 304 304 304 F F F 9 180-199 178 178 178 303 303 303 F F F 10 200-219 180 180 180 308 308 308 F F F 11 220-239 179 179 179 306 306 306 F F F 12 240-259 178 178 178 305 305 305 F F F 13 260-279 178 178 178 305 305 305 F F F 14 280-299 181 181 181 309 309 309 F F F 15 300-319 180 180 180 307 307 307 F F F 16 320-339 177 177 177 302 302 302 F F F 17 340-359 177 177 177 302 302 302 F F F 18 360-379 178 178 178 304 304 304 F F F 19 380-399 179 179 179 306 306 306 F F F 20 400-419 178 178 178 303 303 303 F F F 21 420-439 181 181 181 309 309 309 F F F 22 440-459 181 181 181 309 309 309 F F F 23 460-479 181 181 181 309 309 309 F F F 24 480-499 181 181 181 309 309 309 F F F 25 500-519 170 170 170 289 289 289 F F F 26 520-539 175 175 175 299 299 299 F F F 27 540-559 176 176 176 301 301 301 F F F 28 560-579 172 172 172 294 294 294 F F F 29 580-599 178 178 178 304 304 304 F F F Max gen: 181, Max flush: 309 Begin loop 91 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:02.063148647Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1867,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:02.071206229Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1867,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:02.07150544Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1867} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:02.07153195Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1867} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:02.07154064Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1867} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:02.07155246Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1867} {"msg":"Crucible 50651884-15cf-4f9d-9690-bc38bbbebb5a has session id: 287bf7bf-90d3-4936-bd8d-369fb883c09f","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:02.07189553Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1867,"session_id":"287bf7bf-90d3-4936-bd8d-369fb883c09f"} {"msg":"Upstairs opts: Upstairs UUID: 50651884-15cf-4f9d-9690-bc38bbbebb5a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:02.07191633Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1867,"session_id":"287bf7bf-90d3-4936-bd8d-369fb883c09f"} {"msg":"Crucible stats registered with UUID: 50651884-15cf-4f9d-9690-bc38bbbebb5a","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:02.07192615Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1867,"session_id":"287bf7bf-90d3-4936-bd8d-369fb883c09f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:26:02.07193519Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1867,"session_id":"287bf7bf-90d3-4936-bd8d-369fb883c09f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:02.072088301Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1867,"":"io task","client":"2","":"downstairs","session_id":"287bf7bf-90d3-4936-bd8d-369fb883c09f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:02.072121871Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1867,"":"io task","client":"0","":"downstairs","session_id":"287bf7bf-90d3-4936-bd8d-369fb883c09f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:02.072132961Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1867,"":"io task","client":"1","":"downstairs","session_id":"287bf7bf-90d3-4936-bd8d-369fb883c09f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:02.072247841Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1867,"task":"crutest"} {"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:02.072263541Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1867} {"msg":"50651884-15cf-4f9d-9690-bc38bbbebb5a active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:02.072300091Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1867,"session_id":"287bf7bf-90d3-4936-bd8d-369fb883c09f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:02.072309581Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1867,"client":"0","":"downstairs","session_id":"287bf7bf-90d3-4936-bd8d-369fb883c09f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:02.072319371Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1867,"client":"1","":"downstairs","session_id":"287bf7bf-90d3-4936-bd8d-369fb883c09f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:02.072329191Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1867,"client":"2","":"downstairs","session_id":"287bf7bf-90d3-4936-bd8d-369fb883c09f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:02.072338991Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1867,"":"io task","client":"2","":"downstairs","session_id":"287bf7bf-90d3-4936-bd8d-369fb883c09f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:02.072349081Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1867,"":"io task","client":"0","":"downstairs","session_id":"287bf7bf-90d3-4936-bd8d-369fb883c09f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:02.072359331Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1867,"":"io task","client":"1","":"downstairs","session_id":"287bf7bf-90d3-4936-bd8d-369fb883c09f"} {"msg":"ds_connection connected from Ok(127.0.0.1:48925)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:02.072391861Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1867,"":"io task","client":"0","":"downstairs","session_id":"287bf7bf-90d3-4936-bd8d-369fb883c09f"} {"msg":"ds_connection connected from Ok(127.0.0.1:49749)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:02.072402521Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1867,"":"io task","client":"2","":"downstairs","session_id":"287bf7bf-90d3-4936-bd8d-369fb883c09f"} {"msg":"ds_connection connected from Ok(127.0.0.1:36620)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:02.072560651Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1867,"":"io task","client":"1","":"downstairs","session_id":"287bf7bf-90d3-4936-bd8d-369fb883c09f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:02.072979152Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1867,"client":"0","":"downstairs","session_id":"287bf7bf-90d3-4936-bd8d-369fb883c09f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:02.073046532Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1867,"client":"2","":"downstairs","session_id":"287bf7bf-90d3-4936-bd8d-369fb883c09f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:02.073067682Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1867,"client":"1","":"downstairs","session_id":"287bf7bf-90d3-4936-bd8d-369fb883c09f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:02.073314852Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1867,"session_id":"287bf7bf-90d3-4936-bd8d-369fb883c09f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:02.073344882Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1867,"session_id":"287bf7bf-90d3-4936-bd8d-369fb883c09f"} {"msg":"[0]R flush_numbers[0..12]: [297, 302, 309, 309, 309, 306, 309, 308, 304, 303, 308, 306]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:02.073400393Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1867,"":"downstairs","":"downstairs","session_id":"287bf7bf-90d3-4936-bd8d-369fb883c09f"} {"msg":"[0]R generation[0..12]: [174, 177, 181, 181, 181, 179, 181, 180, 178, 178, 180, 179]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:02.073412383Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1867,"":"downstairs","":"downstairs","session_id":"287bf7bf-90d3-4936-bd8d-369fb883c09f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:02.073423693Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1867,"":"downstairs","":"downstairs","session_id":"287bf7bf-90d3-4936-bd8d-369fb883c09f"} {"msg":"[1]R flush_numbers[0..12]: [297, 302, 309, 309, 309, 306, 309, 308, 304, 303, 308, 306]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:02.073434593Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1867,"":"downstairs","":"downstairs","session_id":"287bf7bf-90d3-4936-bd8d-369fb883c09f"} {"msg":"[1]R generation[0..12]: [174, 177, 181, 181, 181, 179, 181, 180, 178, 178, 180, 179]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:02.073445413Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1867,"":"downstairs","":"downstairs","session_id":"287bf7bf-90d3-4936-bd8d-369fb883c09f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:02.073456483Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1867,"":"downstairs","":"downstairs","session_id":"287bf7bf-90d3-4936-bd8d-369fb883c09f"} {"msg":"[2]R flush_numbers[0..12]: [297, 302, 309, 309, 309, 306, 309, 308, 304, 303, 308, 306]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:02.073469913Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1867,"":"downstairs","":"downstairs","session_id":"287bf7bf-90d3-4936-bd8d-369fb883c09f"} {"msg":"[2]R generation[0..12]: [174, 177, 181, 181, 181, 179, 181, 180, 178, 178, 180, 179]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:02.073480753Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1867,"":"downstairs","":"downstairs","session_id":"287bf7bf-90d3-4936-bd8d-369fb883c09f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:02.073491943Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1867,"":"downstairs","":"downstairs","session_id":"287bf7bf-90d3-4936-bd8d-369fb883c09f"} {"msg":"Max found gen is 182","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:02.073501503Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1867,"":"downstairs","":"downstairs","session_id":"287bf7bf-90d3-4936-bd8d-369fb883c09f"} {"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:02.073511493Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1867,"":"downstairs","":"downstairs","session_id":"287bf7bf-90d3-4936-bd8d-369fb883c09f"} {"msg":"Next flush: 310","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:02.073520943Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1867,"":"downstairs","":"downstairs","session_id":"287bf7bf-90d3-4936-bd8d-369fb883c09f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:02.073530373Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1867,"":"downstairs","":"downstairs","session_id":"287bf7bf-90d3-4936-bd8d-369fb883c09f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:02.073540333Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1867,"session_id":"287bf7bf-90d3-4936-bd8d-369fb883c09f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:02.073549273Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1867,"client":"0","":"downstairs","session_id":"287bf7bf-90d3-4936-bd8d-369fb883c09f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:02.073559163Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1867,"client":"1","":"downstairs","session_id":"287bf7bf-90d3-4936-bd8d-369fb883c09f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:02.073568883Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1867,"client":"2","":"downstairs","session_id":"287bf7bf-90d3-4936-bd8d-369fb883c09f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:02.073578713Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1867,"session_id":"287bf7bf-90d3-4936-bd8d-369fb883c09f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:02.073588023Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1867,"session_id":"287bf7bf-90d3-4936-bd8d-369fb883c09f"} {"msg":"50651884-15cf-4f9d-9690-bc38bbbebb5a is now active with session: 287bf7bf-90d3-4936-bd8d-369fb883c09f","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:02.073598193Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1867,"session_id":"287bf7bf-90d3-4936-bd8d-369fb883c09f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:02.073607093Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1867,"session_id":"287bf7bf-90d3-4936-bd8d-369fb883c09f"} {"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:02.073616393Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1867} {"msg":"Activated sub_volume 50651884-15cf-4f9d-9690-bc38bbbebb5a","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:02.073625973Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1867} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:02.073634023Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1867,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:02.073931143Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1867,"task":"crutest"} 01/30 Write block 3 len 8192 data: 11 11 02/30 Read block 479 len 12288 03/30 Read block 9 len 4096 04/30 Write block 555 len 32768 data: 9 11 14 10 11 12 13 12 05/30 Write block 119 len 8192 data: 9 11 06/30 Read block 589 len 4096 07/30 Write block 394 len 4096 data: 13 08/30 Write block 48 len 16384 data: 11 11 14 14 09/30 Read block 180 len 40960 10/30 Read block 460 len 8192 11/30 Read block 365 len 12288 12/30 Read block 556 len 20480 13/30 Read block 221 len 12288 14/30 Read block 227 len 4096 15/30 Read block 230 len 16384 16/30 Read block 326 len 32768 17/30 Write block 71 len 8192 data: 13 10 18/30 Read block 207 len 40960 19/30 Read block 31 len 16384 20/30 Read block 286 len 20480 21/30 Write block 439 len 36864 data: 13 10 9 11 11 10 10 10 11 22/30 Read block 156 len 24576 23/30 Read block 574 len 20480 24/30 Write block 272 len 8192 data: 21 15 25/30 Read block 432 len 36864 26/30 Read block 51 len 20480 27/30 Write block 187 len 20480 data: 18 17 15 14 16 28/30 Read block 333 len 8192 29/30 Write block 383 len 4096 data: 14 30/30 Read block 550 len 12288 ---------------------------------------------------------------- Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Read 3 Done Done Done false 1002 Acked Read 1 Done Sent Done false 1003 Acked Write 8 Done Sent Done false 1004 Acked Write 2 Done Sent Done false 1005 Acked Read 1 Done Sent Done false 1006 Acked Write 1 Done Sent Done false 1007 Acked Write 4 Done Sent Done false 1008 Acked Read 10 Done Sent Done false 1009 Acked Read 2 Done Sent Done false 1010 Acked Read 3 Done Sent Done false 1011 Acked Read 5 Done Sent Done false 1012 Acked Read 3 Done Sent Done false 1013 Acked Read 1 Done Sent Done false 1014 Acked Read 4 Done Sent Done false 1015 Acked Read 8 Done Sent Done false 1016 Acked Write 2 Done Sent Done false 1017 Acked Read 10 Done Sent Done false 1018 Acked Read 4 Done Sent Done false 1019 Acked Read 5 Done Sent Done false 1020 Acked Write 9 Done Sent Done false 1021 Acked Read 6 Done Sent Done false 1022 Acked Read 5 Done Sent Done false 1023 Acked Write 2 Done Sent Done false 1024 Acked Read 9 Done Sent Done false 1025 Acked Read 5 Done Sent Done false 1026 Acked Write 5 Done Sent Done false 1027 Acked Read 2 Done Sent Done false 1028 Acked Write 1 Done Sent Done false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 28 1 29 Done 30 2 29 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:26:02.081614455Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1867} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:02.081807085Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1867,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 174 174 174 297 297 297 T T T 1 020-039 177 177 177 302 302 302 F F F 2 040-059 181 181 181 309 309 309 T F T <--- 3 060-079 181 181 181 309 309 309 T F T <--- 4 080-099 181 181 181 309 309 309 F F F 5 100-119 179 179 179 306 306 306 T F T <--- 6 120-139 181 181 181 309 309 309 T F T <--- 7 140-159 180 180 180 308 308 308 F F F 8 160-179 178 178 178 304 304 304 F F F 9 180-199 178 178 178 303 303 303 T F T <--- 10 200-219 180 180 180 308 308 308 F F F 11 220-239 179 179 179 306 306 306 F F F 12 240-259 178 178 178 305 305 305 F F F 13 260-279 178 178 178 305 305 305 T F T <--- 14 280-299 181 181 181 309 309 309 F F F 15 300-319 180 180 180 307 307 307 F F F 16 320-339 177 177 177 302 302 302 F F F 17 340-359 177 177 177 302 302 302 F F F 18 360-379 178 178 178 304 304 304 F F F 19 380-399 179 179 179 306 306 306 T F T <--- 20 400-419 178 178 178 303 303 303 F F F 21 420-439 181 181 181 309 309 309 T F T <--- 22 440-459 181 181 181 309 309 309 T F T <--- 23 460-479 181 181 181 309 309 309 F F F 24 480-499 181 181 181 309 309 309 F F F 25 500-519 170 170 170 289 289 289 F F F 26 520-539 175 175 175 299 299 299 F F F 27 540-559 176 176 176 301 301 301 T F T <--- 28 560-579 172 172 172 294 294 294 T F T <--- 29 580-599 178 178 178 304 304 304 F F F Max gen: 181, Max flush: 309 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 91 Verifying data now No change after verify Loop: 91 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 182 182 182 310 310 310 F F F 1 020-039 177 177 177 302 302 302 F F F 2 040-059 182 182 182 310 310 310 F F F 3 060-079 182 182 182 310 310 310 F F F 4 080-099 181 181 181 309 309 309 F F F 5 100-119 182 182 182 310 310 310 F F F 6 120-139 182 182 182 310 310 310 F F F 7 140-159 180 180 180 308 308 308 F F F 8 160-179 178 178 178 304 304 304 F F F 9 180-199 182 182 182 310 310 310 F F F 10 200-219 180 180 180 308 308 308 F F F 11 220-239 179 179 179 306 306 306 F F F 12 240-259 178 178 178 305 305 305 F F F 13 260-279 182 182 182 310 310 310 F F F 14 280-299 181 181 181 309 309 309 F F F 15 300-319 180 180 180 307 307 307 F F F 16 320-339 177 177 177 302 302 302 F F F 17 340-359 177 177 177 302 302 302 F F F 18 360-379 178 178 178 304 304 304 F F F 19 380-399 182 182 182 310 310 310 F F F 20 400-419 178 178 178 303 303 303 F F F 21 420-439 182 182 182 310 310 310 F F F 22 440-459 182 182 182 310 310 310 F F F 23 460-479 181 181 181 309 309 309 F F F 24 480-499 181 181 181 309 309 309 F F F 25 500-519 170 170 170 289 289 289 F F F 26 520-539 175 175 175 299 299 299 F F F 27 540-559 182 182 182 310 310 310 F F F 28 560-579 182 182 182 310 310 310 F F F 29 580-599 178 178 178 304 304 304 F F F Max gen: 182, Max flush: 310 Begin loop 92 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:06.334192922Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1875,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:26:06.341339003Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1875,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:06.341391123Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1875,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:06.348480404Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1875,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:06.348753644Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1875} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:06.348802484Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1875} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:06.348811424Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1875} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:06.348823514Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1875} {"msg":"Crucible ed36de25-19ea-4f97-8a2d-8581170dafed has session id: 02dcd25b-68b3-41e5-881f-370cd5ac15c3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:06.349138315Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1875,"session_id":"02dcd25b-68b3-41e5-881f-370cd5ac15c3"} {"msg":"Upstairs opts: Upstairs UUID: ed36de25-19ea-4f97-8a2d-8581170dafed, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:06.349166825Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1875,"session_id":"02dcd25b-68b3-41e5-881f-370cd5ac15c3"} {"msg":"Crucible stats registered with UUID: ed36de25-19ea-4f97-8a2d-8581170dafed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:06.349177865Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1875,"session_id":"02dcd25b-68b3-41e5-881f-370cd5ac15c3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:26:06.349187035Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1875,"session_id":"02dcd25b-68b3-41e5-881f-370cd5ac15c3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:06.349317545Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1875,"":"io task","client":"2","":"downstairs","session_id":"02dcd25b-68b3-41e5-881f-370cd5ac15c3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:06.349348975Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1875,"":"io task","client":"0","":"downstairs","session_id":"02dcd25b-68b3-41e5-881f-370cd5ac15c3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:06.349360195Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1875,"":"io task","client":"1","":"downstairs","session_id":"02dcd25b-68b3-41e5-881f-370cd5ac15c3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:06.349481995Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1875,"task":"crutest"} {"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:06.349506305Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1875} {"msg":"ed36de25-19ea-4f97-8a2d-8581170dafed active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:06.349524765Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1875,"session_id":"02dcd25b-68b3-41e5-881f-370cd5ac15c3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:06.349533615Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1875,"client":"0","":"downstairs","session_id":"02dcd25b-68b3-41e5-881f-370cd5ac15c3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:06.349543255Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1875,"client":"1","":"downstairs","session_id":"02dcd25b-68b3-41e5-881f-370cd5ac15c3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:06.349553015Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1875,"client":"2","":"downstairs","session_id":"02dcd25b-68b3-41e5-881f-370cd5ac15c3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:06.349562675Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1875,"":"io task","client":"2","":"downstairs","session_id":"02dcd25b-68b3-41e5-881f-370cd5ac15c3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:06.349572895Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1875,"":"io task","client":"0","":"downstairs","session_id":"02dcd25b-68b3-41e5-881f-370cd5ac15c3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:06.349582855Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1875,"":"io task","client":"1","":"downstairs","session_id":"02dcd25b-68b3-41e5-881f-370cd5ac15c3"} {"msg":"ds_connection connected from Ok(127.0.0.1:48749)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:06.349625745Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1875,"":"io task","client":"2","":"downstairs","session_id":"02dcd25b-68b3-41e5-881f-370cd5ac15c3"} {"msg":"ds_connection connected from Ok(127.0.0.1:42995)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:06.349643016Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1875,"":"io task","client":"1","":"downstairs","session_id":"02dcd25b-68b3-41e5-881f-370cd5ac15c3"} {"msg":"ds_connection connected from Ok(127.0.0.1:59023)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:06.349707656Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1875,"":"io task","client":"0","":"downstairs","session_id":"02dcd25b-68b3-41e5-881f-370cd5ac15c3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:06.350288546Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1875,"client":"1","":"downstairs","session_id":"02dcd25b-68b3-41e5-881f-370cd5ac15c3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:06.350312977Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1875,"client":"2","":"downstairs","session_id":"02dcd25b-68b3-41e5-881f-370cd5ac15c3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:06.350324587Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1875,"client":"0","":"downstairs","session_id":"02dcd25b-68b3-41e5-881f-370cd5ac15c3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:06.350615527Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1875,"session_id":"02dcd25b-68b3-41e5-881f-370cd5ac15c3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:06.350625347Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1875,"session_id":"02dcd25b-68b3-41e5-881f-370cd5ac15c3"} {"msg":"[0]R flush_numbers[0..12]: [310, 302, 310, 310, 309, 310, 310, 308, 304, 310, 308, 306]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:06.350644587Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1875,"":"downstairs","":"downstairs","session_id":"02dcd25b-68b3-41e5-881f-370cd5ac15c3"} {"msg":"[0]R generation[0..12]: [182, 177, 182, 182, 181, 182, 182, 180, 178, 182, 180, 179]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:06.350655497Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1875,"":"downstairs","":"downstairs","session_id":"02dcd25b-68b3-41e5-881f-370cd5ac15c3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:06.350666587Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1875,"":"downstairs","":"downstairs","session_id":"02dcd25b-68b3-41e5-881f-370cd5ac15c3"} {"msg":"[1]R flush_numbers[0..12]: [310, 302, 310, 310, 309, 310, 310, 308, 304, 310, 308, 306]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:06.350677397Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1875,"":"downstairs","":"downstairs","session_id":"02dcd25b-68b3-41e5-881f-370cd5ac15c3"} {"msg":"[1]R generation[0..12]: [182, 177, 182, 182, 181, 182, 182, 180, 178, 182, 180, 179]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:06.350688157Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1875,"":"downstairs","":"downstairs","session_id":"02dcd25b-68b3-41e5-881f-370cd5ac15c3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:06.350699187Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1875,"":"downstairs","":"downstairs","session_id":"02dcd25b-68b3-41e5-881f-370cd5ac15c3"} {"msg":"[2]R flush_numbers[0..12]: [310, 302, 310, 310, 309, 310, 310, 308, 304, 310, 308, 306]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:06.350710137Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1875,"":"downstairs","":"downstairs","session_id":"02dcd25b-68b3-41e5-881f-370cd5ac15c3"} {"msg":"[2]R generation[0..12]: [182, 177, 182, 182, 181, 182, 182, 180, 178, 182, 180, 179]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:06.350720997Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1875,"":"downstairs","":"downstairs","session_id":"02dcd25b-68b3-41e5-881f-370cd5ac15c3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:06.350732157Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1875,"":"downstairs","":"downstairs","session_id":"02dcd25b-68b3-41e5-881f-370cd5ac15c3"} {"msg":"Max found gen is 183","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:06.350741777Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1875,"":"downstairs","":"downstairs","session_id":"02dcd25b-68b3-41e5-881f-370cd5ac15c3"} {"msg":"Generation requested: 184 >= found:183","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:06.350753447Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1875,"":"downstairs","":"downstairs","session_id":"02dcd25b-68b3-41e5-881f-370cd5ac15c3"} {"msg":"Next flush: 311","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:06.350762867Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1875,"":"downstairs","":"downstairs","session_id":"02dcd25b-68b3-41e5-881f-370cd5ac15c3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:06.350772507Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1875,"":"downstairs","":"downstairs","session_id":"02dcd25b-68b3-41e5-881f-370cd5ac15c3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:06.350782437Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1875,"session_id":"02dcd25b-68b3-41e5-881f-370cd5ac15c3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:06.350791327Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1875,"client":"0","":"downstairs","session_id":"02dcd25b-68b3-41e5-881f-370cd5ac15c3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:06.350801187Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1875,"client":"1","":"downstairs","session_id":"02dcd25b-68b3-41e5-881f-370cd5ac15c3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:06.350810977Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1875,"client":"2","":"downstairs","session_id":"02dcd25b-68b3-41e5-881f-370cd5ac15c3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:06.350820837Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1875,"session_id":"02dcd25b-68b3-41e5-881f-370cd5ac15c3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:06.350830087Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1875,"session_id":"02dcd25b-68b3-41e5-881f-370cd5ac15c3"} {"msg":"ed36de25-19ea-4f97-8a2d-8581170dafed is now active with session: 02dcd25b-68b3-41e5-881f-370cd5ac15c3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:06.350840237Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1875,"session_id":"02dcd25b-68b3-41e5-881f-370cd5ac15c3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:06.350849107Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1875,"session_id":"02dcd25b-68b3-41e5-881f-370cd5ac15c3"} {"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:06.350858447Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1875} {"msg":"Activated sub_volume ed36de25-19ea-4f97-8a2d-8581170dafed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:06.350866607Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1875} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:06.350874917Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1875,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:06.351172698Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1875,"task":"crutest"} 01/30 Write block 49 len 4096 data: 12 02/30 Write block 87 len 16384 data: 16 17 19 19 03/30 Write block 68 len 8192 data: 12 14 04/30 Write block 591 len 12288 data: 9 10 8 05/30 Read block 270 len 24576 06/30 Read block 86 len 20480 07/30 Write block 123 len 16384 data: 13 12 14 12 08/30 Read block 117 len 16384 09/30 Write block 564 len 32768 data: 11 14 15 17 15 16 16 16 10/30 Write block 344 len 20480 data: 15 14 13 15 13 11/30 Write block 38 len 24576 data: 9 11 9 11 11 11 12/30 Read block 213 len 36864 13/30 Write block 443 len 8192 data: 12 11 14/30 Write block 204 len 16384 data: 14 13 12 12 15/30 Write block 436 len 4096 data: 8 16/30 Read block 442 len 28672 17/30 Read block 407 len 8192 18/30 Write block 589 len 16384 data: 13 9 10 11 19/30 Read block 400 len 32768 20/30 Write block 180 len 36864 data: 12 11 9 11 13 17 16 19 18 21/30 Flush 22/30 Write block 156 len 24576 data: 16 16 19 16 15 14 23/30 Read block 424 len 8192 24/30 Read block 258 len 36864 25/30 Write block 200 len 32768 data: 11 12 12 15 15 14 13 13 26/30 Write block 1 len 36864 data: 6 7 12 12 11 13 12 13 11 27/30 Write block 276 len 12288 data: 18 19 14 28/30 Read block 336 len 24576 29/30 Write block 41 len 24576 data: 12 12 12 7 7 9 30/30 Read block 556 len 28672 ---------------------------------------------------------------- Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Write 4 Done Done Done false 1002 Acked Write 2 Done Done Done false 1003 Acked Write 3 Done Done Done false 1004 Acked Read 6 Done Done Done false 1005 Acked Read 5 Done Done Done false 1006 Acked Write 4 Done Done Done false 1007 Acked Read 4 Done Done Done false 1008 Acked Write 8 Sent Done Done false 1009 Acked Write 5 Sent Done Done false 1010 Acked Write 6 Sent Done Done false 1011 Acked Read 9 Sent Done Done false 1012 Acked Write 2 Sent Done Done false 1013 Acked Write 4 Sent Done Done false 1014 Acked Write 1 Sent Done Done false 1015 Acked Read 7 Sent Done Done false 1016 Acked Read 2 Sent Done Done false 1017 Acked Write 4 Sent Done Done false 1018 Acked Read 8 Sent Done Done false 1019 Acked Write 9 Sent Done Done false 1020 Acked Flush 0 Sent Done Done false 1021 Acked Write 6 Sent Done Done false 1022 Acked Read 2 Sent Done Done false 1023 Acked Read 9 Sent Done Done false 1024 Acked Write 8 Sent Done Done false 1025 Acked Write 9 Sent Done Done false 1026 Acked Write 3 Sent Done Done false 1027 Acked Read 6 Sent Done Done false 1028 Acked Write 6 Sent Done Done false 1029 Acked Read 7 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 22 0 1 23 Done 8 30 29 67 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:26:06.35936509Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1875} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:06.35955895Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1875,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 182 182 182 310 310 310 F T T <--- 1 020-039 177 184 184 302 311 311 F F F <--- 2 040-059 182 184 184 310 311 311 T T T <--- 3 060-079 182 184 184 310 311 311 T F F <--- 4 080-099 181 184 184 309 311 311 T F F <--- 5 100-119 182 182 182 310 310 310 F F F 6 120-139 182 184 184 310 311 311 T F F <--- 7 140-159 180 180 180 308 308 308 F T T <--- 8 160-179 178 178 178 304 304 304 F T T <--- 9 180-199 182 184 184 310 311 311 F F F <--- 10 200-219 180 184 184 308 311 311 F T T <--- 11 220-239 179 179 179 306 306 306 F F F 12 240-259 178 178 178 305 305 305 F F F 13 260-279 182 182 182 310 310 310 F T T <--- 14 280-299 181 181 181 309 309 309 F F F 15 300-319 180 180 180 307 307 307 F F F 16 320-339 177 177 177 302 302 302 F F F 17 340-359 177 184 184 302 311 311 F F F <--- 18 360-379 178 178 178 304 304 304 F F F 19 380-399 182 182 182 310 310 310 F F F 20 400-419 178 178 178 303 303 303 F F F 21 420-439 182 184 184 310 311 311 F F F <--- 22 440-459 182 184 184 310 311 311 F F F <--- 23 460-479 181 181 181 309 309 309 F F F 24 480-499 181 181 181 309 309 309 F F F 25 500-519 170 170 170 289 289 289 F F F 26 520-539 175 175 175 299 299 299 F F F 27 540-559 182 182 182 310 310 310 F F F 28 560-579 182 184 184 310 311 311 F F F <--- 29 580-599 178 184 184 304 311 311 T F F <--- Max gen: 184, Max flush: 311 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 92 Verifying data now No change after verify Loop: 92 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 185 185 185 312 312 312 F F F 1 020-039 184 184 184 311 311 311 F F F 2 040-059 185 185 185 312 312 312 F F F 3 060-079 184 184 184 311 311 311 F F F 4 080-099 184 184 184 311 311 311 F F F 5 100-119 182 182 182 310 310 310 F F F 6 120-139 184 184 184 311 311 311 F F F 7 140-159 185 185 185 312 312 312 F F F 8 160-179 185 185 185 312 312 312 F F F 9 180-199 184 184 184 311 311 311 F F F 10 200-219 185 185 185 312 312 312 F F F 11 220-239 179 179 179 306 306 306 F F F 12 240-259 178 178 178 305 305 305 F F F 13 260-279 185 185 185 312 312 312 F F F 14 280-299 181 181 181 309 309 309 F F F 15 300-319 180 180 180 307 307 307 F F F 16 320-339 177 177 177 302 302 302 F F F 17 340-359 184 184 184 311 311 311 F F F 18 360-379 178 178 178 304 304 304 F F F 19 380-399 182 182 182 310 310 310 F F F 20 400-419 178 178 178 303 303 303 F F F 21 420-439 184 184 184 311 311 311 F F F 22 440-459 184 184 184 311 311 311 F F F 23 460-479 181 181 181 309 309 309 F F F 24 480-499 181 181 181 309 309 309 F F F 25 500-519 170 170 170 289 289 289 F F F 26 520-539 175 175 175 299 299 299 F F F 27 540-559 182 182 182 310 310 310 F F F 28 560-579 184 184 184 311 311 311 F F F 29 580-599 184 184 184 311 311 311 F F F Max gen: 185, Max flush: 312 Begin loop 93 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:10.625047797Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1883,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:10.6335283Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1883,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:10.633846851Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1883} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:10.633873881Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1883} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:10.633882511Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1883} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:10.633894361Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1883} {"msg":"Crucible 0b97a129-cc6a-4773-ad8d-faf51e2883b9 has session id: a5919025-6768-40d1-a5b5-24af841edebb","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:10.634233331Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1883,"session_id":"a5919025-6768-40d1-a5b5-24af841edebb"} {"msg":"Upstairs opts: Upstairs UUID: 0b97a129-cc6a-4773-ad8d-faf51e2883b9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:10.634256541Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1883,"session_id":"a5919025-6768-40d1-a5b5-24af841edebb"} {"msg":"Crucible stats registered with UUID: 0b97a129-cc6a-4773-ad8d-faf51e2883b9","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:10.634266741Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1883,"session_id":"a5919025-6768-40d1-a5b5-24af841edebb"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:26:10.634275881Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1883,"session_id":"a5919025-6768-40d1-a5b5-24af841edebb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:10.634418051Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1883,"":"io task","client":"1","":"downstairs","session_id":"a5919025-6768-40d1-a5b5-24af841edebb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:10.634449641Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1883,"":"io task","client":"0","":"downstairs","session_id":"a5919025-6768-40d1-a5b5-24af841edebb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:10.634460731Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1883,"":"io task","client":"2","":"downstairs","session_id":"a5919025-6768-40d1-a5b5-24af841edebb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:10.634579802Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1883,"task":"crutest"} {"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:10.634593812Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1883} {"msg":"0b97a129-cc6a-4773-ad8d-faf51e2883b9 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:10.634636082Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1883,"session_id":"a5919025-6768-40d1-a5b5-24af841edebb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:10.634645772Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1883,"client":"0","":"downstairs","session_id":"a5919025-6768-40d1-a5b5-24af841edebb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:10.634655982Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1883,"client":"1","":"downstairs","session_id":"a5919025-6768-40d1-a5b5-24af841edebb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:10.634665822Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1883,"client":"2","":"downstairs","session_id":"a5919025-6768-40d1-a5b5-24af841edebb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:10.634675652Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1883,"":"io task","client":"2","":"downstairs","session_id":"a5919025-6768-40d1-a5b5-24af841edebb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:10.634685862Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1883,"":"io task","client":"0","":"downstairs","session_id":"a5919025-6768-40d1-a5b5-24af841edebb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:10.634695962Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1883,"":"io task","client":"1","":"downstairs","session_id":"a5919025-6768-40d1-a5b5-24af841edebb"} {"msg":"ds_connection connected from Ok(127.0.0.1:48371)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:10.634764742Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1883,"":"io task","client":"0","":"downstairs","session_id":"a5919025-6768-40d1-a5b5-24af841edebb"} {"msg":"ds_connection connected from Ok(127.0.0.1:45435)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:10.634789742Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1883,"":"io task","client":"2","":"downstairs","session_id":"a5919025-6768-40d1-a5b5-24af841edebb"} {"msg":"ds_connection connected from Ok(127.0.0.1:37755)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:10.634800972Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1883,"":"io task","client":"1","":"downstairs","session_id":"a5919025-6768-40d1-a5b5-24af841edebb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:10.635461063Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1883,"client":"2","":"downstairs","session_id":"a5919025-6768-40d1-a5b5-24af841edebb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:10.635546133Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1883,"client":"0","":"downstairs","session_id":"a5919025-6768-40d1-a5b5-24af841edebb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:10.635599353Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1883,"client":"1","":"downstairs","session_id":"a5919025-6768-40d1-a5b5-24af841edebb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:10.635810663Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1883,"session_id":"a5919025-6768-40d1-a5b5-24af841edebb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:10.635830073Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1883,"session_id":"a5919025-6768-40d1-a5b5-24af841edebb"} {"msg":"[0]R flush_numbers[0..12]: [312, 311, 312, 311, 311, 310, 311, 312, 312, 311, 312, 306]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:10.635890214Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1883,"":"downstairs","":"downstairs","session_id":"a5919025-6768-40d1-a5b5-24af841edebb"} {"msg":"[0]R generation[0..12]: [185, 184, 185, 184, 184, 182, 184, 185, 185, 184, 185, 179]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:10.635904054Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1883,"":"downstairs","":"downstairs","session_id":"a5919025-6768-40d1-a5b5-24af841edebb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:10.635915444Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1883,"":"downstairs","":"downstairs","session_id":"a5919025-6768-40d1-a5b5-24af841edebb"} {"msg":"[1]R flush_numbers[0..12]: [312, 311, 312, 311, 311, 310, 311, 312, 312, 311, 312, 306]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:10.635926174Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1883,"":"downstairs","":"downstairs","session_id":"a5919025-6768-40d1-a5b5-24af841edebb"} {"msg":"[1]R generation[0..12]: [185, 184, 185, 184, 184, 182, 184, 185, 185, 184, 185, 179]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:10.635936864Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1883,"":"downstairs","":"downstairs","session_id":"a5919025-6768-40d1-a5b5-24af841edebb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:10.635947924Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1883,"":"downstairs","":"downstairs","session_id":"a5919025-6768-40d1-a5b5-24af841edebb"} {"msg":"[2]R flush_numbers[0..12]: [312, 311, 312, 311, 311, 310, 311, 312, 312, 311, 312, 306]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:10.635958814Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1883,"":"downstairs","":"downstairs","session_id":"a5919025-6768-40d1-a5b5-24af841edebb"} {"msg":"[2]R generation[0..12]: [185, 184, 185, 184, 184, 182, 184, 185, 185, 184, 185, 179]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:10.635969514Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1883,"":"downstairs","":"downstairs","session_id":"a5919025-6768-40d1-a5b5-24af841edebb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:10.635980594Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1883,"":"downstairs","":"downstairs","session_id":"a5919025-6768-40d1-a5b5-24af841edebb"} {"msg":"Max found gen is 186","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:10.635990154Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1883,"":"downstairs","":"downstairs","session_id":"a5919025-6768-40d1-a5b5-24af841edebb"} {"msg":"Generation requested: 186 >= found:186","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:10.635999904Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1883,"":"downstairs","":"downstairs","session_id":"a5919025-6768-40d1-a5b5-24af841edebb"} {"msg":"Next flush: 313","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:10.636009444Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1883,"":"downstairs","":"downstairs","session_id":"a5919025-6768-40d1-a5b5-24af841edebb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:10.636019024Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1883,"":"downstairs","":"downstairs","session_id":"a5919025-6768-40d1-a5b5-24af841edebb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:10.636028914Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1883,"session_id":"a5919025-6768-40d1-a5b5-24af841edebb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:10.636037694Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1883,"client":"0","":"downstairs","session_id":"a5919025-6768-40d1-a5b5-24af841edebb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:10.636048484Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1883,"client":"1","":"downstairs","session_id":"a5919025-6768-40d1-a5b5-24af841edebb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:10.636058344Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1883,"client":"2","":"downstairs","session_id":"a5919025-6768-40d1-a5b5-24af841edebb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:10.636068304Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1883,"session_id":"a5919025-6768-40d1-a5b5-24af841edebb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:10.636077574Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1883,"session_id":"a5919025-6768-40d1-a5b5-24af841edebb"} {"msg":"0b97a129-cc6a-4773-ad8d-faf51e2883b9 is now active with session: a5919025-6768-40d1-a5b5-24af841edebb","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:10.636087754Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1883,"session_id":"a5919025-6768-40d1-a5b5-24af841edebb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:10.636096544Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1883,"session_id":"a5919025-6768-40d1-a5b5-24af841edebb"} {"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:10.636105674Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1883} {"msg":"Activated sub_volume 0b97a129-cc6a-4773-ad8d-faf51e2883b9","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:10.636113794Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1883} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:10.636122004Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1883,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:10.636395194Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1883,"task":"crutest"} 01/30 Write block 528 len 24576 data: 19 15 12 12 12 13 02/30 Write block 362 len 4096 data: 15 03/30 Read block 217 len 28672 04/30 Read block 93 len 32768 05/30 Write block 31 len 12288 data: 17 14 12 06/30 Write block 502 len 12288 data: 16 14 11 07/30 Read block 363 len 12288 08/30 Write block 246 len 28672 data: 14 14 14 16 15 18 13 09/30 Read block 88 len 24576 10/30 Write block 564 len 28672 data: 12 15 16 18 16 17 17 11/30 Flush 12/30 Write block 464 len 28672 data: 19 17 15 15 16 13 12 13/30 Read block 246 len 40960 14/30 Write block 326 len 32768 data: 12 12 13 9 13 16 16 14 15/30 Write block 540 len 4096 data: 11 16/30 Read block 434 len 28672 17/30 Read block 314 len 12288 18/30 Write block 70 len 28672 data: 16 14 11 10 9 9 11 19/30 Read block 47 len 24576 20/30 Write block 51 len 40960 data: 15 16 15 13 14 10 12 15 16 16 21/30 Write block 107 len 24576 data: 12 13 14 15 15 15 22/30 Write block 513 len 16384 data: 12 11 11 11 23/30 Write block 92 len 28672 data: 18 16 16 15 13 14 14 24/30 Read block 391 len 16384 25/30 Read block 208 len 20480 26/30 Write block 86 len 16384 data: 17 17 18 20 27/30 Write block 572 len 32768 data: 14 15 16 17 15 15 16 14 28/30 Read block 325 len 28672 29/30 Write block 14 len 24576 data: 13 12 12 12 12 15 30/30 Read block 454 len 24576 ---------------------------------------------------------------- Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Sent Done false 1001 Acked Write 1 Done Sent Done false 1002 Acked Read 7 Done Sent Done false 1003 Acked Read 8 Done Sent Done false 1004 Acked Write 3 Done Sent Done false 1005 Acked Write 3 Done Sent Done false 1006 Acked Read 3 Done Sent Done false 1007 Acked Write 7 Done Sent Done false 1008 Acked Read 6 Done Sent Done false 1009 Acked Write 7 Done Sent Done false 1010 Acked Flush 0 Done Sent Done false 1011 Acked Write 7 Done Sent Done false 1012 Acked Read 10 Done Sent Done false 1013 Acked Write 8 Done Sent Done false 1014 Acked Write 1 Done Sent Done false 1015 Acked Read 7 Done Sent Done false 1016 Acked Read 3 Done Sent Done false 1017 Acked Write 7 Done Sent Done false 1018 Acked Read 6 Done Sent Done false 1019 Acked Write 10 Done Sent Done false 1020 Acked Write 6 Done Sent Done false 1021 Acked Write 4 Done Sent Done false 1022 Acked Write 7 Done Sent Done false 1023 Acked Read 4 Done Sent Done false 1024 Acked Read 5 Done Sent Done false 1025 Acked Write 4 Done Sent Done false 1026 Acked Write 8 Done Sent Done false 1027 Acked Read 7 Done Sent Done false 1028 Acked Write 6 Done Sent Done false 1029 Acked Read 6 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 30 0 31 Done 29 0 30 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1010)) None Some(JobId(1010)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:26:10.645529638Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1883} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:10.645724558Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1883,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 185 185 185 312 312 312 T F T <--- 1 020-039 186 184 186 313 311 313 F F F <--- 2 040-059 185 185 185 312 312 312 T F T <--- 3 060-079 184 184 184 311 311 311 T F T <--- 4 080-099 184 184 184 311 311 311 T F T <--- 5 100-119 182 182 182 310 310 310 T F T <--- 6 120-139 184 184 184 311 311 311 F F F 7 140-159 185 185 185 312 312 312 F F F 8 160-179 185 185 185 312 312 312 F F F 9 180-199 184 184 184 311 311 311 F F F 10 200-219 185 185 185 312 312 312 F F F 11 220-239 179 179 179 306 306 306 F F F 12 240-259 186 178 186 313 305 313 F F F <--- 13 260-279 185 185 185 312 312 312 F F F 14 280-299 181 181 181 309 309 309 F F F 15 300-319 180 180 180 307 307 307 F F F 16 320-339 177 177 177 302 302 302 T F T <--- 17 340-359 184 184 184 311 311 311 F F F 18 360-379 186 178 186 313 304 313 F F F <--- 19 380-399 182 182 182 310 310 310 F F F 20 400-419 178 178 178 303 303 303 F F F 21 420-439 184 184 184 311 311 311 F F F 22 440-459 184 184 184 311 311 311 F F F 23 460-479 181 181 181 309 309 309 T F T <--- 24 480-499 181 181 181 309 309 309 F F F 25 500-519 186 170 186 313 289 313 T F T <--- 26 520-539 186 175 186 313 299 313 F F F <--- 27 540-559 182 182 182 310 310 310 T F T <--- 28 560-579 186 184 186 313 311 313 T F T <--- 29 580-599 184 184 184 311 311 311 F F F Max gen: 186, Max flush: 313 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 93 Verifying data now No change after verify Loop: 93 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 187 187 187 314 314 314 F F F 1 020-039 186 186 186 313 313 313 F F F 2 040-059 187 187 187 314 314 314 F F F 3 060-079 187 187 187 314 314 314 F F F 4 080-099 187 187 187 314 314 314 F F F 5 100-119 187 187 187 314 314 314 F F F 6 120-139 184 184 184 311 311 311 F F F 7 140-159 185 185 185 312 312 312 F F F 8 160-179 185 185 185 312 312 312 F F F 9 180-199 184 184 184 311 311 311 F F F 10 200-219 185 185 185 312 312 312 F F F 11 220-239 179 179 179 306 306 306 F F F 12 240-259 186 186 186 313 313 313 F F F 13 260-279 185 185 185 312 312 312 F F F 14 280-299 181 181 181 309 309 309 F F F 15 300-319 180 180 180 307 307 307 F F F 16 320-339 187 187 187 314 314 314 F F F 17 340-359 184 184 184 311 311 311 F F F 18 360-379 186 186 186 313 313 313 F F F 19 380-399 182 182 182 310 310 310 F F F 20 400-419 178 178 178 303 303 303 F F F 21 420-439 184 184 184 311 311 311 F F F 22 440-459 184 184 184 311 311 311 F F F 23 460-479 187 187 187 314 314 314 F F F 24 480-499 181 181 181 309 309 309 F F F 25 500-519 187 187 187 314 314 314 F F F 26 520-539 186 186 186 313 313 313 F F F 27 540-559 187 187 187 314 314 314 F F F 28 560-579 187 187 187 314 314 314 F F F 29 580-599 184 184 184 311 311 311 F F F Max gen: 187, Max flush: 314 Begin loop 94 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:14.903406533Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1891,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:14.911571505Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1891,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:14.911871346Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1891} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:14.911900356Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1891} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:14.911909186Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1891} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:14.911923656Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1891} {"msg":"Crucible 6814b92f-1d50-4791-968d-fd22b23b78e1 has session id: 68d9d434-fef2-4b5d-a769-fe46c4c7251f","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:14.912262516Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1891,"session_id":"68d9d434-fef2-4b5d-a769-fe46c4c7251f"} {"msg":"Upstairs opts: Upstairs UUID: 6814b92f-1d50-4791-968d-fd22b23b78e1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:14.912284216Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1891,"session_id":"68d9d434-fef2-4b5d-a769-fe46c4c7251f"} {"msg":"Crucible stats registered with UUID: 6814b92f-1d50-4791-968d-fd22b23b78e1","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:14.912294466Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1891,"session_id":"68d9d434-fef2-4b5d-a769-fe46c4c7251f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:26:14.912303626Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1891,"session_id":"68d9d434-fef2-4b5d-a769-fe46c4c7251f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:14.912455707Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1891,"":"io task","client":"1","":"downstairs","session_id":"68d9d434-fef2-4b5d-a769-fe46c4c7251f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:14.912487467Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1891,"":"io task","client":"0","":"downstairs","session_id":"68d9d434-fef2-4b5d-a769-fe46c4c7251f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:14.912498467Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1891,"":"io task","client":"2","":"downstairs","session_id":"68d9d434-fef2-4b5d-a769-fe46c4c7251f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:14.912618977Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1891,"task":"crutest"} {"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:14.912631027Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1891} {"msg":"6814b92f-1d50-4791-968d-fd22b23b78e1 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:14.912665427Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1891,"session_id":"68d9d434-fef2-4b5d-a769-fe46c4c7251f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:14.912674967Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1891,"client":"0","":"downstairs","session_id":"68d9d434-fef2-4b5d-a769-fe46c4c7251f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:14.912684967Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1891,"client":"1","":"downstairs","session_id":"68d9d434-fef2-4b5d-a769-fe46c4c7251f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:14.912694847Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1891,"client":"2","":"downstairs","session_id":"68d9d434-fef2-4b5d-a769-fe46c4c7251f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:14.912704807Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1891,"":"io task","client":"2","":"downstairs","session_id":"68d9d434-fef2-4b5d-a769-fe46c4c7251f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:14.912717157Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1891,"":"io task","client":"0","":"downstairs","session_id":"68d9d434-fef2-4b5d-a769-fe46c4c7251f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:14.912727397Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1891,"":"io task","client":"1","":"downstairs","session_id":"68d9d434-fef2-4b5d-a769-fe46c4c7251f"} {"msg":"ds_connection connected from Ok(127.0.0.1:38601)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:14.912817547Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1891,"":"io task","client":"1","":"downstairs","session_id":"68d9d434-fef2-4b5d-a769-fe46c4c7251f"} {"msg":"ds_connection connected from Ok(127.0.0.1:36752)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:14.912895207Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1891,"":"io task","client":"2","":"downstairs","session_id":"68d9d434-fef2-4b5d-a769-fe46c4c7251f"} {"msg":"ds_connection connected from Ok(127.0.0.1:58103)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:14.912915587Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1891,"":"io task","client":"0","":"downstairs","session_id":"68d9d434-fef2-4b5d-a769-fe46c4c7251f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:14.913460318Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1891,"client":"2","":"downstairs","session_id":"68d9d434-fef2-4b5d-a769-fe46c4c7251f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:14.913504518Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1891,"client":"1","":"downstairs","session_id":"68d9d434-fef2-4b5d-a769-fe46c4c7251f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:14.913522208Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1891,"client":"0","":"downstairs","session_id":"68d9d434-fef2-4b5d-a769-fe46c4c7251f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:14.913793319Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1891,"session_id":"68d9d434-fef2-4b5d-a769-fe46c4c7251f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:14.913838559Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1891,"session_id":"68d9d434-fef2-4b5d-a769-fe46c4c7251f"} {"msg":"[0]R flush_numbers[0..12]: [314, 313, 314, 314, 314, 314, 311, 312, 312, 311, 312, 306]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:14.913897179Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1891,"":"downstairs","":"downstairs","session_id":"68d9d434-fef2-4b5d-a769-fe46c4c7251f"} {"msg":"[0]R generation[0..12]: [187, 186, 187, 187, 187, 187, 184, 185, 185, 184, 185, 179]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:14.913908549Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1891,"":"downstairs","":"downstairs","session_id":"68d9d434-fef2-4b5d-a769-fe46c4c7251f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:14.913919699Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1891,"":"downstairs","":"downstairs","session_id":"68d9d434-fef2-4b5d-a769-fe46c4c7251f"} {"msg":"[1]R flush_numbers[0..12]: [314, 313, 314, 314, 314, 314, 311, 312, 312, 311, 312, 306]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:14.913930469Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1891,"":"downstairs","":"downstairs","session_id":"68d9d434-fef2-4b5d-a769-fe46c4c7251f"} {"msg":"[1]R generation[0..12]: [187, 186, 187, 187, 187, 187, 184, 185, 185, 184, 185, 179]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:14.913944099Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1891,"":"downstairs","":"downstairs","session_id":"68d9d434-fef2-4b5d-a769-fe46c4c7251f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:14.913955169Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1891,"":"downstairs","":"downstairs","session_id":"68d9d434-fef2-4b5d-a769-fe46c4c7251f"} {"msg":"[2]R flush_numbers[0..12]: [314, 313, 314, 314, 314, 314, 311, 312, 312, 311, 312, 306]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:14.913965889Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1891,"":"downstairs","":"downstairs","session_id":"68d9d434-fef2-4b5d-a769-fe46c4c7251f"} {"msg":"[2]R generation[0..12]: [187, 186, 187, 187, 187, 187, 184, 185, 185, 184, 185, 179]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:14.913976469Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1891,"":"downstairs","":"downstairs","session_id":"68d9d434-fef2-4b5d-a769-fe46c4c7251f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:14.913987659Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1891,"":"downstairs","":"downstairs","session_id":"68d9d434-fef2-4b5d-a769-fe46c4c7251f"} {"msg":"Max found gen is 188","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:14.913997009Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1891,"":"downstairs","":"downstairs","session_id":"68d9d434-fef2-4b5d-a769-fe46c4c7251f"} {"msg":"Generation requested: 188 >= found:188","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:14.914006869Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1891,"":"downstairs","":"downstairs","session_id":"68d9d434-fef2-4b5d-a769-fe46c4c7251f"} {"msg":"Next flush: 315","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:14.914016399Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1891,"":"downstairs","":"downstairs","session_id":"68d9d434-fef2-4b5d-a769-fe46c4c7251f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:14.914025919Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1891,"":"downstairs","":"downstairs","session_id":"68d9d434-fef2-4b5d-a769-fe46c4c7251f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:14.914035609Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1891,"session_id":"68d9d434-fef2-4b5d-a769-fe46c4c7251f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:14.914044539Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1891,"client":"0","":"downstairs","session_id":"68d9d434-fef2-4b5d-a769-fe46c4c7251f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:14.914054299Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1891,"client":"1","":"downstairs","session_id":"68d9d434-fef2-4b5d-a769-fe46c4c7251f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:14.914064099Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1891,"client":"2","":"downstairs","session_id":"68d9d434-fef2-4b5d-a769-fe46c4c7251f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:14.914073939Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1891,"session_id":"68d9d434-fef2-4b5d-a769-fe46c4c7251f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:14.914083189Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1891,"session_id":"68d9d434-fef2-4b5d-a769-fe46c4c7251f"} {"msg":"6814b92f-1d50-4791-968d-fd22b23b78e1 is now active with session: 68d9d434-fef2-4b5d-a769-fe46c4c7251f","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:14.914093229Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1891,"session_id":"68d9d434-fef2-4b5d-a769-fe46c4c7251f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:14.914103369Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1891,"session_id":"68d9d434-fef2-4b5d-a769-fe46c4c7251f"} {"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:14.914112509Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1891} {"msg":"Activated sub_volume 6814b92f-1d50-4791-968d-fd22b23b78e1","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:14.914120539Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1891} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:14.914128599Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1891,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:14.91447264Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1891,"task":"crutest"} 01/30 Write block 141 len 28672 data: 14 12 10 12 10 9 10 02/30 Write block 585 len 36864 data: 16 17 19 17 14 10 11 12 9 03/30 Read block 125 len 8192 04/30 Write block 520 len 12288 data: 15 15 13 05/30 Read block 15 len 32768 06/30 Write block 412 len 20480 data: 17 18 15 15 16 07/30 Flush 08/30 Write block 345 len 24576 data: 15 14 16 14 13 9 09/30 Write block 517 len 28672 data: 11 11 14 16 16 14 12 10/30 Write block 307 len 32768 data: 15 10 10 12 10 9 9 9 11/30 Read block 325 len 32768 12/30 Write block 297 len 28672 data: 17 19 19 17 18 19 17 13/30 Write block 298 len 8192 data: 20 20 14/30 Write block 557 len 20480 data: 15 11 12 13 14 15/30 Read block 220 len 8192 16/30 Read block 547 len 28672 17/30 Write block 585 len 40960 data: 17 18 20 18 15 11 12 13 10 8 18/30 Read block 56 len 12288 19/30 Read block 72 len 36864 20/30 Write block 419 len 12288 data: 13 15 15 21/30 Read block 49 len 20480 22/30 Write block 200 len 8192 data: 12 13 23/30 Write block 227 len 20480 data: 13 13 9 9 8 24/30 Write block 62 len 32768 data: 19 19 17 16 15 15 13 15 25/30 Write block 341 len 16384 data: 18 14 14 16 26/30 Write block 8 len 12288 data: 14 12 12 27/30 Write block 146 len 16384 data: 10 11 10 12 28/30 Read block 507 len 4096 29/30 Write block 247 len 28672 data: 15 15 17 16 19 14 11 30/30 Write block 564 len 36864 data: 13 16 17 19 17 18 18 17 15 ---------------------------------------------------------------- Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Write 9 Done Done Done false 1002 Acked Read 2 Done Sent Done false 1003 Acked Write 3 Done Sent Done false 1004 Acked Read 8 Done Sent Done false 1005 Acked Write 5 Done Sent Done false 1006 Acked Flush 0 Done Sent Done false 1007 Acked Write 6 Done Sent Done false 1008 Acked Write 7 Done Sent Done false 1009 Acked Write 8 Done Sent Done false 1010 Acked Read 8 Done Sent Done false 1011 Acked Write 7 Done Sent Done false 1012 Acked Write 2 Done Sent Done false 1013 Acked Write 5 Done Sent Done false 1014 Acked Read 2 Done Sent Done false 1015 Acked Read 7 Done Sent Done false 1016 Acked Write 10 Done Sent Done false 1017 Acked Read 3 Done Sent Done false 1018 Acked Read 9 Done Sent Done false 1019 Acked Write 3 Done Sent Done false 1020 Acked Read 5 Done Sent Done false 1021 Acked Write 2 Done Sent Done false 1022 Acked Write 5 Done Sent Done false 1023 Acked Write 8 Done Sent Done false 1024 Acked Write 4 Done Sent Done false 1025 Acked Write 3 Done Sent Done false 1026 Acked Write 4 Done Sent Done false 1027 Acked Read 1 Done Sent Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 28 2 32 Done 28 2 28 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1006)) None Some(JobId(1006)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:26:14.922465082Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1891} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:14.922682712Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1891,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 187 187 187 314 314 314 T F T <--- 1 020-039 186 186 186 313 313 313 F F F 2 040-059 187 187 187 314 314 314 F F F 3 060-079 187 187 187 314 314 314 T F T <--- 4 080-099 187 187 187 314 314 314 F F F 5 100-119 187 187 187 314 314 314 F F F 6 120-139 184 184 184 311 311 311 F F F 7 140-159 188 185 188 315 312 315 T T T <--- 8 160-179 185 185 185 312 312 312 F F F 9 180-199 184 184 184 311 311 311 F F F 10 200-219 185 185 185 312 312 312 T F T <--- 11 220-239 179 179 179 306 306 306 T F T <--- 12 240-259 186 186 186 313 313 313 T F T <--- 13 260-279 185 185 185 312 312 312 F F F 14 280-299 181 181 181 309 309 309 T F T <--- 15 300-319 180 180 180 307 307 307 T F T <--- 16 320-339 187 187 187 314 314 314 F F F 17 340-359 184 184 184 311 311 311 T F T <--- 18 360-379 186 186 186 313 313 313 F F F 19 380-399 182 182 182 310 310 310 F F F 20 400-419 188 178 188 315 303 315 T F T <--- 21 420-439 184 184 184 311 311 311 T F T <--- 22 440-459 184 184 184 311 311 311 F F F 23 460-479 187 187 187 314 314 314 F F F 24 480-499 181 181 181 309 309 309 F F F 25 500-519 187 187 187 314 314 314 T F T <--- 26 520-539 188 186 188 315 313 315 T F T <--- 27 540-559 187 187 187 314 314 314 T F T <--- 28 560-579 187 187 187 314 314 314 T F T <--- 29 580-599 188 184 188 315 311 315 T T T <--- Max gen: 188, Max flush: 315 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 94 Verifying data now No change after verify Loop: 94 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 189 189 189 316 316 316 F F F 1 020-039 186 186 186 313 313 313 F F F 2 040-059 187 187 187 314 314 314 F F F 3 060-079 189 189 189 316 316 316 F F F 4 080-099 187 187 187 314 314 314 F F F 5 100-119 187 187 187 314 314 314 F F F 6 120-139 184 184 184 311 311 311 F F F 7 140-159 189 189 189 316 316 316 F F F 8 160-179 185 185 185 312 312 312 F F F 9 180-199 184 184 184 311 311 311 F F F 10 200-219 189 189 189 316 316 316 F F F 11 220-239 189 189 189 316 316 316 F F F 12 240-259 189 189 189 316 316 316 F F F 13 260-279 185 185 185 312 312 312 F F F 14 280-299 189 189 189 316 316 316 F F F 15 300-319 189 189 189 316 316 316 F F F 16 320-339 187 187 187 314 314 314 F F F 17 340-359 189 189 189 316 316 316 F F F 18 360-379 186 186 186 313 313 313 F F F 19 380-399 182 182 182 310 310 310 F F F 20 400-419 189 189 189 316 316 316 F F F 21 420-439 189 189 189 316 316 316 F F F 22 440-459 184 184 184 311 311 311 F F F 23 460-479 187 187 187 314 314 314 F F F 24 480-499 181 181 181 309 309 309 F F F 25 500-519 189 189 189 316 316 316 F F F 26 520-539 189 189 189 316 316 316 F F F 27 540-559 189 189 189 316 316 316 F F F 28 560-579 189 189 189 316 316 316 F F F 29 580-599 189 189 189 316 316 316 F F F Max gen: 189, Max flush: 316 Begin loop 95 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:19.188106959Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1899,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:26:19.195244079Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1899,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:19.1952985Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1899,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:19.20216775Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1899,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:19.20243148Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1899} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:19.20246702Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1899} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:19.20247635Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1899} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:19.20248835Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1899} {"msg":"Crucible abde2d11-29d4-4ec2-be8a-f224ffbe5f15 has session id: 343999a3-8f52-4b12-94a5-104643f8296a","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:19.202821731Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1899,"session_id":"343999a3-8f52-4b12-94a5-104643f8296a"} {"msg":"Upstairs opts: Upstairs UUID: abde2d11-29d4-4ec2-be8a-f224ffbe5f15, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:19.202845001Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1899,"session_id":"343999a3-8f52-4b12-94a5-104643f8296a"} {"msg":"Crucible stats registered with UUID: abde2d11-29d4-4ec2-be8a-f224ffbe5f15","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:19.202855261Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1899,"session_id":"343999a3-8f52-4b12-94a5-104643f8296a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:26:19.202864351Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1899,"session_id":"343999a3-8f52-4b12-94a5-104643f8296a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:19.202989141Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1899,"":"io task","client":"2","":"downstairs","session_id":"343999a3-8f52-4b12-94a5-104643f8296a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:19.203013911Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1899,"":"io task","client":"0","":"downstairs","session_id":"343999a3-8f52-4b12-94a5-104643f8296a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:19.203024131Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1899,"":"io task","client":"1","":"downstairs","session_id":"343999a3-8f52-4b12-94a5-104643f8296a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:19.203184041Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1899,"task":"crutest"} {"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:19.203202841Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1899} {"msg":"abde2d11-29d4-4ec2-be8a-f224ffbe5f15 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:19.203232561Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1899,"session_id":"343999a3-8f52-4b12-94a5-104643f8296a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:19.203242092Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1899,"client":"0","":"downstairs","session_id":"343999a3-8f52-4b12-94a5-104643f8296a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:19.203251792Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1899,"client":"1","":"downstairs","session_id":"343999a3-8f52-4b12-94a5-104643f8296a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:19.203261492Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1899,"client":"2","":"downstairs","session_id":"343999a3-8f52-4b12-94a5-104643f8296a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:19.203271112Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1899,"":"io task","client":"2","":"downstairs","session_id":"343999a3-8f52-4b12-94a5-104643f8296a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:19.203281082Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1899,"":"io task","client":"0","":"downstairs","session_id":"343999a3-8f52-4b12-94a5-104643f8296a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:19.203291062Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1899,"":"io task","client":"1","":"downstairs","session_id":"343999a3-8f52-4b12-94a5-104643f8296a"} {"msg":"ds_connection connected from Ok(127.0.0.1:41698)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:19.203351272Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1899,"":"io task","client":"0","":"downstairs","session_id":"343999a3-8f52-4b12-94a5-104643f8296a"} {"msg":"ds_connection connected from Ok(127.0.0.1:44170)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:19.203382732Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1899,"":"io task","client":"2","":"downstairs","session_id":"343999a3-8f52-4b12-94a5-104643f8296a"} {"msg":"ds_connection connected from Ok(127.0.0.1:48652)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:19.203394312Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1899,"":"io task","client":"1","":"downstairs","session_id":"343999a3-8f52-4b12-94a5-104643f8296a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:19.204046773Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1899,"client":"1","":"downstairs","session_id":"343999a3-8f52-4b12-94a5-104643f8296a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:19.204104133Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1899,"client":"2","":"downstairs","session_id":"343999a3-8f52-4b12-94a5-104643f8296a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:19.204177713Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1899,"client":"0","":"downstairs","session_id":"343999a3-8f52-4b12-94a5-104643f8296a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:19.204359453Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1899,"session_id":"343999a3-8f52-4b12-94a5-104643f8296a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:19.204384363Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1899,"session_id":"343999a3-8f52-4b12-94a5-104643f8296a"} {"msg":"[0]R flush_numbers[0..12]: [316, 313, 314, 316, 314, 314, 311, 316, 312, 311, 316, 316]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:19.204471853Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1899,"":"downstairs","":"downstairs","session_id":"343999a3-8f52-4b12-94a5-104643f8296a"} {"msg":"[0]R generation[0..12]: [189, 186, 187, 189, 187, 187, 184, 189, 185, 184, 189, 189]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:19.204483333Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1899,"":"downstairs","":"downstairs","session_id":"343999a3-8f52-4b12-94a5-104643f8296a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:19.204494473Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1899,"":"downstairs","":"downstairs","session_id":"343999a3-8f52-4b12-94a5-104643f8296a"} {"msg":"[1]R flush_numbers[0..12]: [316, 313, 314, 316, 314, 314, 311, 316, 312, 311, 316, 316]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:19.204510463Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1899,"":"downstairs","":"downstairs","session_id":"343999a3-8f52-4b12-94a5-104643f8296a"} {"msg":"[1]R generation[0..12]: [189, 186, 187, 189, 187, 187, 184, 189, 185, 184, 189, 189]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:19.204521353Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1899,"":"downstairs","":"downstairs","session_id":"343999a3-8f52-4b12-94a5-104643f8296a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:19.204532573Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1899,"":"downstairs","":"downstairs","session_id":"343999a3-8f52-4b12-94a5-104643f8296a"} {"msg":"[2]R flush_numbers[0..12]: [316, 313, 314, 316, 314, 314, 311, 316, 312, 311, 316, 316]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:19.204548013Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1899,"":"downstairs","":"downstairs","session_id":"343999a3-8f52-4b12-94a5-104643f8296a"} {"msg":"[2]R generation[0..12]: [189, 186, 187, 189, 187, 187, 184, 189, 185, 184, 189, 189]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:19.204560203Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1899,"":"downstairs","":"downstairs","session_id":"343999a3-8f52-4b12-94a5-104643f8296a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:19.204571254Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1899,"":"downstairs","":"downstairs","session_id":"343999a3-8f52-4b12-94a5-104643f8296a"} {"msg":"Max found gen is 190","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:19.204580774Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1899,"":"downstairs","":"downstairs","session_id":"343999a3-8f52-4b12-94a5-104643f8296a"} {"msg":"Generation requested: 190 >= found:190","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:19.204590504Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1899,"":"downstairs","":"downstairs","session_id":"343999a3-8f52-4b12-94a5-104643f8296a"} {"msg":"Next flush: 317","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:19.204600014Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1899,"":"downstairs","":"downstairs","session_id":"343999a3-8f52-4b12-94a5-104643f8296a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:19.204609464Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1899,"":"downstairs","":"downstairs","session_id":"343999a3-8f52-4b12-94a5-104643f8296a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:19.204619624Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1899,"session_id":"343999a3-8f52-4b12-94a5-104643f8296a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:19.204628534Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1899,"client":"0","":"downstairs","session_id":"343999a3-8f52-4b12-94a5-104643f8296a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:19.204638594Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1899,"client":"1","":"downstairs","session_id":"343999a3-8f52-4b12-94a5-104643f8296a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:19.204648404Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1899,"client":"2","":"downstairs","session_id":"343999a3-8f52-4b12-94a5-104643f8296a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:19.204658534Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1899,"session_id":"343999a3-8f52-4b12-94a5-104643f8296a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:19.204667884Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1899,"session_id":"343999a3-8f52-4b12-94a5-104643f8296a"} {"msg":"abde2d11-29d4-4ec2-be8a-f224ffbe5f15 is now active with session: 343999a3-8f52-4b12-94a5-104643f8296a","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:19.204678144Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1899,"session_id":"343999a3-8f52-4b12-94a5-104643f8296a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:19.204686934Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1899,"session_id":"343999a3-8f52-4b12-94a5-104643f8296a"} {"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:19.204696194Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1899} {"msg":"Activated sub_volume abde2d11-29d4-4ec2-be8a-f224ffbe5f15","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:19.204705804Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1899} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:19.204714004Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1899,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:19.205023294Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1899,"task":"crutest"} 01/30 Write block 538 len 32768 data: 12 12 12 12 13 17 20 16 02/30 Write block 78 len 8192 data: 17 18 03/30 Write block 363 len 20480 data: 18 19 18 18 14 04/30 Read block 355 len 32768 05/30 Read block 282 len 28672 06/30 Write block 389 len 36864 data: 12 12 12 13 11 14 13 12 10 07/30 Read block 468 len 28672 08/30 Write block 493 len 16384 data: 16 15 14 13 09/30 Read block 33 len 20480 10/30 Read block 160 len 24576 11/30 Read block 67 len 8192 12/30 Flush 13/30 Write block 51 len 12288 data: 16 17 16 14/30 Write block 284 len 36864 data: 11 14 14 13 13 12 13 11 13 15/30 Write block 145 len 16384 data: 11 11 12 11 16/30 Flush 17/30 Write block 469 len 8192 data: 14 13 18/30 Write block 57 len 28672 data: 13 16 17 17 19 20 20 19/30 Read block 575 len 20480 20/30 Write block 593 len 24576 data: 11 9 10 8 6 5 21/30 Write block 347 len 40960 data: 17 15 14 10 6 9 10 12 12 11 22/30 Write block 166 len 28672 data: 10 13 16 14 15 16 17 23/30 Read block 182 len 20480 24/30 Read block 511 len 40960 25/30 Write block 109 len 40960 data: 15 16 16 16 18 17 18 17 12 10 26/30 Read block 111 len 20480 27/30 Read block 37 len 8192 28/30 Read block 452 len 40960 29/30 Read block 487 len 4096 30/30 Write block 507 len 8192 data: 10 10 ---------------------------------------------------------------- Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Write 2 Done Done Done false 1002 Acked Write 5 Sent Done Done false 1003 Acked Read 8 Sent Done Done false 1004 Acked Read 7 Sent Done Done false 1005 Acked Write 9 Sent Done Done false 1006 Acked Read 7 Sent Done Done false 1007 Acked Write 4 Sent Done Done false 1008 Acked Read 5 Sent Done Done false 1009 Acked Read 6 Sent Done Done false 1010 Acked Read 2 Sent Done Done false 1011 Acked Flush 0 Sent Done Done false 1012 Acked Write 3 Sent Done Done false 1013 Acked Write 9 Sent Done Done false 1014 Acked Write 4 Sent Done Done false 1015 Acked Flush 0 Sent Done Done false 1016 Acked Write 2 Sent Done Done false 1017 Acked Write 7 Sent Done Done false 1018 Acked Read 5 Sent Done Done false 1019 Acked Write 6 Sent Done Done false 1020 Acked Write 10 Sent Done Done false 1021 Acked Write 7 Sent Done Done false 1022 Acked Read 5 Sent Done Done false 1023 Acked Read 10 Sent Done Done false 1024 Acked Write 10 Sent Done Done false 1025 Acked Read 5 Sent Done Done false 1026 Acked Read 2 Sent Done Done false 1027 Acked Read 10 Sent Done Done false 1028 Acked Read 1 Sent Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 1 1 30 Done 2 29 29 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1015)) Some(JobId(1015)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:26:19.214177038Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1899} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:19.214354518Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1899,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 189 189 189 316 316 316 F F F 1 020-039 186 186 186 313 313 313 F F F 2 040-059 187 190 190 314 318 318 F T T <--- 3 060-079 189 190 190 316 317 317 T T T <--- 4 080-099 187 187 187 314 314 314 F F F 5 100-119 187 187 187 314 314 314 F T T <--- 6 120-139 184 184 184 311 311 311 F F F 7 140-159 189 190 190 316 318 318 F F F <--- 8 160-179 185 185 185 312 312 312 F T T <--- 9 180-199 184 184 184 311 311 311 F F F 10 200-219 189 189 189 316 316 316 F F F 11 220-239 189 189 189 316 316 316 F F F 12 240-259 189 189 189 316 316 316 F F F 13 260-279 185 185 185 312 312 312 F F F 14 280-299 189 190 190 316 318 318 F F F <--- 15 300-319 189 189 189 316 316 316 F F F 16 320-339 187 187 187 314 314 314 F F F 17 340-359 189 189 189 316 316 316 F T T <--- 18 360-379 186 190 190 313 317 317 F F F <--- 19 380-399 182 190 190 310 317 317 F F F <--- 20 400-419 189 189 189 316 316 316 F F F 21 420-439 189 189 189 316 316 316 F F F 22 440-459 184 184 184 311 311 311 F F F 23 460-479 187 187 187 314 314 314 F T T <--- 24 480-499 181 190 190 309 317 317 F F F <--- 25 500-519 189 189 189 316 316 316 F T T <--- 26 520-539 189 190 190 316 317 317 T F F <--- 27 540-559 189 190 190 316 317 317 T F F <--- 28 560-579 189 189 189 316 316 316 F F F 29 580-599 189 189 189 316 316 316 F T T <--- Max gen: 190, Max flush: 318 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 95 Verifying data now No change after verify Loop: 95 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 189 189 189 316 316 316 F F F 1 020-039 186 186 186 313 313 313 F F F 2 040-059 191 191 191 319 319 319 F F F 3 060-079 191 191 191 319 319 319 F F F 4 080-099 187 187 187 314 314 314 F F F 5 100-119 191 191 191 319 319 319 F F F 6 120-139 184 184 184 311 311 311 F F F 7 140-159 190 190 190 318 318 318 F F F 8 160-179 191 191 191 319 319 319 F F F 9 180-199 184 184 184 311 311 311 F F F 10 200-219 189 189 189 316 316 316 F F F 11 220-239 189 189 189 316 316 316 F F F 12 240-259 189 189 189 316 316 316 F F F 13 260-279 185 185 185 312 312 312 F F F 14 280-299 190 190 190 318 318 318 F F F 15 300-319 189 189 189 316 316 316 F F F 16 320-339 187 187 187 314 314 314 F F F 17 340-359 191 191 191 319 319 319 F F F 18 360-379 190 190 190 317 317 317 F F F 19 380-399 190 190 190 317 317 317 F F F 20 400-419 189 189 189 316 316 316 F F F 21 420-439 189 189 189 316 316 316 F F F 22 440-459 184 184 184 311 311 311 F F F 23 460-479 191 191 191 319 319 319 F F F 24 480-499 190 190 190 317 317 317 F F F 25 500-519 191 191 191 319 319 319 F F F 26 520-539 190 190 190 317 317 317 F F F 27 540-559 190 190 190 317 317 317 F F F 28 560-579 189 189 189 316 316 316 F F F 29 580-599 191 191 191 319 319 319 F F F Max gen: 191, Max flush: 319 Begin loop 96 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:23.475500658Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1907,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:23.483813431Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1907,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:23.484111471Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1907} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:23.484140042Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1907} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:23.484148802Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1907} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:23.484160742Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1907} {"msg":"Crucible 18c16298-f7a4-44c6-8d3e-b9e32479ce2a has session id: e26da9d3-c5e2-4f5a-9336-7fdf81c150fe","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:23.484518672Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1907,"session_id":"e26da9d3-c5e2-4f5a-9336-7fdf81c150fe"} {"msg":"Upstairs opts: Upstairs UUID: 18c16298-f7a4-44c6-8d3e-b9e32479ce2a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:23.484539612Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1907,"session_id":"e26da9d3-c5e2-4f5a-9336-7fdf81c150fe"} {"msg":"Crucible stats registered with UUID: 18c16298-f7a4-44c6-8d3e-b9e32479ce2a","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:23.484549472Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1907,"session_id":"e26da9d3-c5e2-4f5a-9336-7fdf81c150fe"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:26:23.484558482Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1907,"session_id":"e26da9d3-c5e2-4f5a-9336-7fdf81c150fe"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:23.484706872Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1907,"":"io task","client":"0","":"downstairs","session_id":"e26da9d3-c5e2-4f5a-9336-7fdf81c150fe"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:23.484741762Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1907,"":"io task","client":"2","":"downstairs","session_id":"e26da9d3-c5e2-4f5a-9336-7fdf81c150fe"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:23.484752772Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1907,"":"io task","client":"1","":"downstairs","session_id":"e26da9d3-c5e2-4f5a-9336-7fdf81c150fe"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:23.484904413Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1907,"task":"crutest"} {"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:23.484923353Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1907} {"msg":"18c16298-f7a4-44c6-8d3e-b9e32479ce2a active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:23.484947073Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1907,"session_id":"e26da9d3-c5e2-4f5a-9336-7fdf81c150fe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:23.484956553Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1907,"client":"0","":"downstairs","session_id":"e26da9d3-c5e2-4f5a-9336-7fdf81c150fe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:23.484966423Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1907,"client":"1","":"downstairs","session_id":"e26da9d3-c5e2-4f5a-9336-7fdf81c150fe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:23.484976063Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1907,"client":"2","":"downstairs","session_id":"e26da9d3-c5e2-4f5a-9336-7fdf81c150fe"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:23.484985673Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1907,"":"io task","client":"2","":"downstairs","session_id":"e26da9d3-c5e2-4f5a-9336-7fdf81c150fe"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:23.484995633Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1907,"":"io task","client":"0","":"downstairs","session_id":"e26da9d3-c5e2-4f5a-9336-7fdf81c150fe"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:23.485005733Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1907,"":"io task","client":"1","":"downstairs","session_id":"e26da9d3-c5e2-4f5a-9336-7fdf81c150fe"} {"msg":"ds_connection connected from Ok(127.0.0.1:59820)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:23.485081653Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1907,"":"io task","client":"0","":"downstairs","session_id":"e26da9d3-c5e2-4f5a-9336-7fdf81c150fe"} {"msg":"ds_connection connected from Ok(127.0.0.1:52746)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:23.485106693Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1907,"":"io task","client":"2","":"downstairs","session_id":"e26da9d3-c5e2-4f5a-9336-7fdf81c150fe"} {"msg":"ds_connection connected from Ok(127.0.0.1:36344)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:23.485118273Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1907,"":"io task","client":"1","":"downstairs","session_id":"e26da9d3-c5e2-4f5a-9336-7fdf81c150fe"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:23.485814894Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1907,"client":"1","":"downstairs","session_id":"e26da9d3-c5e2-4f5a-9336-7fdf81c150fe"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:23.485832994Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1907,"client":"2","":"downstairs","session_id":"e26da9d3-c5e2-4f5a-9336-7fdf81c150fe"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:23.485846364Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1907,"client":"0","":"downstairs","session_id":"e26da9d3-c5e2-4f5a-9336-7fdf81c150fe"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:23.486139045Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1907,"session_id":"e26da9d3-c5e2-4f5a-9336-7fdf81c150fe"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:23.486161955Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1907,"session_id":"e26da9d3-c5e2-4f5a-9336-7fdf81c150fe"} {"msg":"[0]R flush_numbers[0..12]: [316, 313, 319, 319, 314, 319, 311, 318, 319, 311, 316, 316]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:23.486190305Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1907,"":"downstairs","":"downstairs","session_id":"e26da9d3-c5e2-4f5a-9336-7fdf81c150fe"} {"msg":"[0]R generation[0..12]: [189, 186, 191, 191, 187, 191, 184, 190, 191, 184, 189, 189]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:23.486202185Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1907,"":"downstairs","":"downstairs","session_id":"e26da9d3-c5e2-4f5a-9336-7fdf81c150fe"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:23.486213515Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1907,"":"downstairs","":"downstairs","session_id":"e26da9d3-c5e2-4f5a-9336-7fdf81c150fe"} {"msg":"[1]R flush_numbers[0..12]: [316, 313, 319, 319, 314, 319, 311, 318, 319, 311, 316, 316]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:23.486224385Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1907,"":"downstairs","":"downstairs","session_id":"e26da9d3-c5e2-4f5a-9336-7fdf81c150fe"} {"msg":"[1]R generation[0..12]: [189, 186, 191, 191, 187, 191, 184, 190, 191, 184, 189, 189]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:23.486235135Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1907,"":"downstairs","":"downstairs","session_id":"e26da9d3-c5e2-4f5a-9336-7fdf81c150fe"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:23.486246105Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1907,"":"downstairs","":"downstairs","session_id":"e26da9d3-c5e2-4f5a-9336-7fdf81c150fe"} {"msg":"[2]R flush_numbers[0..12]: [316, 313, 319, 319, 314, 319, 311, 318, 319, 311, 316, 316]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:23.486256945Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1907,"":"downstairs","":"downstairs","session_id":"e26da9d3-c5e2-4f5a-9336-7fdf81c150fe"} {"msg":"[2]R generation[0..12]: [189, 186, 191, 191, 187, 191, 184, 190, 191, 184, 189, 189]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:23.486267705Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1907,"":"downstairs","":"downstairs","session_id":"e26da9d3-c5e2-4f5a-9336-7fdf81c150fe"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:23.486278815Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1907,"":"downstairs","":"downstairs","session_id":"e26da9d3-c5e2-4f5a-9336-7fdf81c150fe"} {"msg":"Max found gen is 192","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:23.486288355Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1907,"":"downstairs","":"downstairs","session_id":"e26da9d3-c5e2-4f5a-9336-7fdf81c150fe"} {"msg":"Generation requested: 192 >= found:192","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:23.486298325Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1907,"":"downstairs","":"downstairs","session_id":"e26da9d3-c5e2-4f5a-9336-7fdf81c150fe"} {"msg":"Next flush: 320","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:23.486309785Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1907,"":"downstairs","":"downstairs","session_id":"e26da9d3-c5e2-4f5a-9336-7fdf81c150fe"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:23.486319385Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1907,"":"downstairs","":"downstairs","session_id":"e26da9d3-c5e2-4f5a-9336-7fdf81c150fe"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:23.486329215Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1907,"session_id":"e26da9d3-c5e2-4f5a-9336-7fdf81c150fe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:23.486338045Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1907,"client":"0","":"downstairs","session_id":"e26da9d3-c5e2-4f5a-9336-7fdf81c150fe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:23.486347705Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1907,"client":"1","":"downstairs","session_id":"e26da9d3-c5e2-4f5a-9336-7fdf81c150fe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:23.486357555Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1907,"client":"2","":"downstairs","session_id":"e26da9d3-c5e2-4f5a-9336-7fdf81c150fe"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:23.486367505Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1907,"session_id":"e26da9d3-c5e2-4f5a-9336-7fdf81c150fe"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:23.486376745Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1907,"session_id":"e26da9d3-c5e2-4f5a-9336-7fdf81c150fe"} {"msg":"18c16298-f7a4-44c6-8d3e-b9e32479ce2a is now active with session: e26da9d3-c5e2-4f5a-9336-7fdf81c150fe","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:23.486386825Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1907,"session_id":"e26da9d3-c5e2-4f5a-9336-7fdf81c150fe"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:23.486395675Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1907,"session_id":"e26da9d3-c5e2-4f5a-9336-7fdf81c150fe"} {"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:23.486404935Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1907} {"msg":"Activated sub_volume 18c16298-f7a4-44c6-8d3e-b9e32479ce2a","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:23.486412905Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1907} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:23.486420995Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1907,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:23.486749635Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1907,"task":"crutest"} 01/30 Write block 452 len 12288 data: 10 10 10 02/30 Write block 595 len 16384 data: 11 9 7 6 03/30 Write block 475 len 12288 data: 8 12 13 04/30 Write block 358 len 40960 data: 12 13 18 14 16 19 20 19 19 15 05/30 Read block 394 len 12288 06/30 Write block 484 len 36864 data: 14 16 17 19 20 20 18 12 14 07/30 Write block 497 len 16384 data: 12 11 15 13 08/30 Read block 468 len 20480 09/30 Read block 142 len 24576 10/30 Read block 112 len 40960 11/30 Read block 459 len 4096 12/30 Write block 305 len 28672 data: 17 16 16 11 11 13 11 13/30 Read block 368 len 36864 14/30 Write block 337 len 12288 data: 15 14 16 15/30 Write block 93 len 16384 data: 17 17 16 14 16/30 Flush 17/30 Flush 18/30 Flush 19/30 Flush 20/30 Write block 331 len 16384 data: 17 17 15 14 21/30 Read block 583 len 36864 22/30 Write block 479 len 28672 data: 10 11 10 10 12 15 17 23/30 Flush 24/30 Write block 570 len 40960 data: 19 18 16 16 17 18 16 16 17 15 25/30 Read block 247 len 32768 26/30 Flush 27/30 Write block 256 len 16384 data: 9 9 15 13 28/30 Write block 438 len 20480 data: 12 14 11 10 12 29/30 Read block 196 len 4096 30/30 Read block 398 len 4096 ---------------------------------------------------------------- Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Write 4 Done Done Done false 1002 Acked Write 3 Done Done Done false 1003 Acked Write 10 Done Sent Done false 1004 Acked Read 3 Done Sent Done false 1005 Acked Write 9 Done Sent Done false 1006 Acked Write 4 Done Sent Done false 1007 Acked Read 5 Done Sent Done false 1008 Acked Read 6 Done Sent Done false 1009 Acked Read 10 Done Sent Done false 1010 Acked Read 1 Done Sent Done false 1011 Acked Write 7 Done Sent Done false 1012 Acked Read 9 Done Sent Done false 1013 Acked Write 3 Done Sent Done false 1014 Acked Write 4 Done Sent Done false 1015 Acked Flush 0 Done Sent Done false 1016 Acked Flush 0 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Write 4 Done Sent Done false 1020 Acked Read 9 Done Sent Done false 1021 Acked Write 7 Done Sent Done false 1022 Acked Flush 0 Done Sent Done false 1023 Acked Write 10 Done Sent Done false 1024 Acked Read 8 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 4 Done Sent Done false 1027 Acked Write 5 Done Sent Done false 1028 Acked Read 1 Done Sent Done false 1029 Acked Read 1 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 27 0 27 Done 30 3 30 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) None Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:26:23.495270818Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1907} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:23.495459129Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1907,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 189 189 189 316 316 316 F F F 1 020-039 186 186 186 313 313 313 F F F 2 040-059 191 191 191 319 319 319 F F F 3 060-079 191 191 191 319 319 319 F F F 4 080-099 192 187 192 320 314 320 F F F <--- 5 100-119 191 191 191 319 319 319 F F F 6 120-139 184 184 184 311 311 311 F F F 7 140-159 190 190 190 318 318 318 F F F 8 160-179 191 191 191 319 319 319 F F F 9 180-199 184 184 184 311 311 311 F F F 10 200-219 189 189 189 316 316 316 F F F 11 220-239 189 189 189 316 316 316 F F F 12 240-259 189 189 189 316 316 316 T F T <--- 13 260-279 185 185 185 312 312 312 F F F 14 280-299 190 190 190 318 318 318 F F F 15 300-319 192 189 192 320 316 320 F F F <--- 16 320-339 192 187 192 324 314 324 F F F <--- 17 340-359 192 191 192 320 319 320 F F F <--- 18 360-379 192 190 192 320 317 320 F F F <--- 19 380-399 190 190 190 317 317 317 F F F 20 400-419 189 189 189 316 316 316 F F F 21 420-439 189 189 189 316 316 316 T F T <--- 22 440-459 192 184 192 320 311 320 T T T <--- 23 460-479 192 191 192 324 319 324 F T F <--- 24 480-499 192 190 192 324 317 324 F F F <--- 25 500-519 192 191 192 320 319 320 F F F <--- 26 520-539 190 190 190 317 317 317 F F F 27 540-559 190 190 190 317 317 317 F F F 28 560-579 192 189 192 325 316 325 F F F <--- 29 580-599 192 191 192 320 319 320 F T F <--- Max gen: 192, Max flush: 325 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 96 Verifying data now No change after verify Loop: 96 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 189 189 189 316 316 316 F F F 1 020-039 186 186 186 313 313 313 F F F 2 040-059 191 191 191 319 319 319 F F F 3 060-079 191 191 191 319 319 319 F F F 4 080-099 192 192 192 320 320 320 F F F 5 100-119 191 191 191 319 319 319 F F F 6 120-139 184 184 184 311 311 311 F F F 7 140-159 190 190 190 318 318 318 F F F 8 160-179 191 191 191 319 319 319 F F F 9 180-199 184 184 184 311 311 311 F F F 10 200-219 189 189 189 316 316 316 F F F 11 220-239 189 189 189 316 316 316 F F F 12 240-259 193 193 193 326 326 326 F F F 13 260-279 185 185 185 312 312 312 F F F 14 280-299 190 190 190 318 318 318 F F F 15 300-319 192 192 192 320 320 320 F F F 16 320-339 192 192 192 324 324 324 F F F 17 340-359 192 192 192 320 320 320 F F F 18 360-379 192 192 192 320 320 320 F F F 19 380-399 190 190 190 317 317 317 F F F 20 400-419 189 189 189 316 316 316 F F F 21 420-439 193 193 193 326 326 326 F F F 22 440-459 193 193 193 326 326 326 F F F 23 460-479 192 192 192 324 324 324 F F F 24 480-499 192 192 192 324 324 324 F F F 25 500-519 192 192 192 320 320 320 F F F 26 520-539 190 190 190 317 317 317 F F F 27 540-559 190 190 190 317 317 317 F F F 28 560-579 192 192 192 325 325 325 F F F 29 580-599 192 192 192 320 320 320 F F F Max gen: 193, Max flush: 326 Begin loop 97 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:27.748026766Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1915,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:26:27.755415197Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1915,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:27.755472147Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1915,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:27.762608458Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1915,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:27.762900458Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1915} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:27.762929818Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1915} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:27.762938258Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1915} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:27.762950118Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1915} {"msg":"Crucible dae107c7-0bd7-420d-a173-2f1902c70ab7 has session id: 5c137e93-49fd-46cd-9f17-1c63b08fb9d7","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:27.763380759Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1915,"session_id":"5c137e93-49fd-46cd-9f17-1c63b08fb9d7"} {"msg":"Upstairs opts: Upstairs UUID: dae107c7-0bd7-420d-a173-2f1902c70ab7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:27.763409679Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1915,"session_id":"5c137e93-49fd-46cd-9f17-1c63b08fb9d7"} {"msg":"Crucible stats registered with UUID: dae107c7-0bd7-420d-a173-2f1902c70ab7","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:27.763420619Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1915,"session_id":"5c137e93-49fd-46cd-9f17-1c63b08fb9d7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:26:27.763429879Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1915,"session_id":"5c137e93-49fd-46cd-9f17-1c63b08fb9d7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:27.763574439Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1915,"":"io task","client":"2","":"downstairs","session_id":"5c137e93-49fd-46cd-9f17-1c63b08fb9d7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:27.763592139Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1915,"":"io task","client":"0","":"downstairs","session_id":"5c137e93-49fd-46cd-9f17-1c63b08fb9d7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:27.763602359Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1915,"":"io task","client":"1","":"downstairs","session_id":"5c137e93-49fd-46cd-9f17-1c63b08fb9d7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:27.76375093Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1915,"task":"crutest"} {"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:27.76377497Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1915} {"msg":"dae107c7-0bd7-420d-a173-2f1902c70ab7 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:27.76378992Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1915,"session_id":"5c137e93-49fd-46cd-9f17-1c63b08fb9d7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:27.76379906Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1915,"client":"0","":"downstairs","session_id":"5c137e93-49fd-46cd-9f17-1c63b08fb9d7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:27.76380897Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1915,"client":"1","":"downstairs","session_id":"5c137e93-49fd-46cd-9f17-1c63b08fb9d7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:27.76381877Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1915,"client":"2","":"downstairs","session_id":"5c137e93-49fd-46cd-9f17-1c63b08fb9d7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:27.76382839Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1915,"":"io task","client":"2","":"downstairs","session_id":"5c137e93-49fd-46cd-9f17-1c63b08fb9d7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:27.76383847Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1915,"":"io task","client":"0","":"downstairs","session_id":"5c137e93-49fd-46cd-9f17-1c63b08fb9d7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:27.76384858Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1915,"":"io task","client":"1","":"downstairs","session_id":"5c137e93-49fd-46cd-9f17-1c63b08fb9d7"} {"msg":"ds_connection connected from Ok(127.0.0.1:47443)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:27.76390083Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1915,"":"io task","client":"1","":"downstairs","session_id":"5c137e93-49fd-46cd-9f17-1c63b08fb9d7"} {"msg":"ds_connection connected from Ok(127.0.0.1:61535)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:27.76391173Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1915,"":"io task","client":"0","":"downstairs","session_id":"5c137e93-49fd-46cd-9f17-1c63b08fb9d7"} {"msg":"ds_connection connected from Ok(127.0.0.1:45830)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:27.76392199Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1915,"":"io task","client":"2","":"downstairs","session_id":"5c137e93-49fd-46cd-9f17-1c63b08fb9d7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:27.764650591Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1915,"client":"1","":"downstairs","session_id":"5c137e93-49fd-46cd-9f17-1c63b08fb9d7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:27.764672581Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1915,"client":"2","":"downstairs","session_id":"5c137e93-49fd-46cd-9f17-1c63b08fb9d7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:27.764683931Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1915,"client":"0","":"downstairs","session_id":"5c137e93-49fd-46cd-9f17-1c63b08fb9d7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:27.764969841Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1915,"session_id":"5c137e93-49fd-46cd-9f17-1c63b08fb9d7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:27.764980041Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1915,"session_id":"5c137e93-49fd-46cd-9f17-1c63b08fb9d7"} {"msg":"[0]R flush_numbers[0..12]: [316, 313, 319, 319, 320, 319, 311, 318, 319, 311, 316, 316]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:27.765029922Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1915,"":"downstairs","":"downstairs","session_id":"5c137e93-49fd-46cd-9f17-1c63b08fb9d7"} {"msg":"[0]R generation[0..12]: [189, 186, 191, 191, 192, 191, 184, 190, 191, 184, 189, 189]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:27.765043602Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1915,"":"downstairs","":"downstairs","session_id":"5c137e93-49fd-46cd-9f17-1c63b08fb9d7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:27.765054952Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1915,"":"downstairs","":"downstairs","session_id":"5c137e93-49fd-46cd-9f17-1c63b08fb9d7"} {"msg":"[1]R flush_numbers[0..12]: [316, 313, 319, 319, 320, 319, 311, 318, 319, 311, 316, 316]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:27.765065842Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1915,"":"downstairs","":"downstairs","session_id":"5c137e93-49fd-46cd-9f17-1c63b08fb9d7"} {"msg":"[1]R generation[0..12]: [189, 186, 191, 191, 192, 191, 184, 190, 191, 184, 189, 189]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:27.765076462Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1915,"":"downstairs","":"downstairs","session_id":"5c137e93-49fd-46cd-9f17-1c63b08fb9d7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:27.765087692Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1915,"":"downstairs","":"downstairs","session_id":"5c137e93-49fd-46cd-9f17-1c63b08fb9d7"} {"msg":"[2]R flush_numbers[0..12]: [316, 313, 319, 319, 320, 319, 311, 318, 319, 311, 316, 316]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:27.765098282Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1915,"":"downstairs","":"downstairs","session_id":"5c137e93-49fd-46cd-9f17-1c63b08fb9d7"} {"msg":"[2]R generation[0..12]: [189, 186, 191, 191, 192, 191, 184, 190, 191, 184, 189, 189]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:27.765109232Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1915,"":"downstairs","":"downstairs","session_id":"5c137e93-49fd-46cd-9f17-1c63b08fb9d7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:27.765120272Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1915,"":"downstairs","":"downstairs","session_id":"5c137e93-49fd-46cd-9f17-1c63b08fb9d7"} {"msg":"Max found gen is 194","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:27.765129812Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1915,"":"downstairs","":"downstairs","session_id":"5c137e93-49fd-46cd-9f17-1c63b08fb9d7"} {"msg":"Generation requested: 194 >= found:194","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:27.765139582Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1915,"":"downstairs","":"downstairs","session_id":"5c137e93-49fd-46cd-9f17-1c63b08fb9d7"} {"msg":"Next flush: 327","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:27.765149082Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1915,"":"downstairs","":"downstairs","session_id":"5c137e93-49fd-46cd-9f17-1c63b08fb9d7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:27.765158562Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1915,"":"downstairs","":"downstairs","session_id":"5c137e93-49fd-46cd-9f17-1c63b08fb9d7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:27.765168332Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1915,"session_id":"5c137e93-49fd-46cd-9f17-1c63b08fb9d7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:27.765177122Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1915,"client":"0","":"downstairs","session_id":"5c137e93-49fd-46cd-9f17-1c63b08fb9d7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:27.765188412Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1915,"client":"1","":"downstairs","session_id":"5c137e93-49fd-46cd-9f17-1c63b08fb9d7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:27.765198222Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1915,"client":"2","":"downstairs","session_id":"5c137e93-49fd-46cd-9f17-1c63b08fb9d7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:27.765208112Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1915,"session_id":"5c137e93-49fd-46cd-9f17-1c63b08fb9d7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:27.765217442Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1915,"session_id":"5c137e93-49fd-46cd-9f17-1c63b08fb9d7"} {"msg":"dae107c7-0bd7-420d-a173-2f1902c70ab7 is now active with session: 5c137e93-49fd-46cd-9f17-1c63b08fb9d7","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:27.765227662Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1915,"session_id":"5c137e93-49fd-46cd-9f17-1c63b08fb9d7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:27.765236512Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1915,"session_id":"5c137e93-49fd-46cd-9f17-1c63b08fb9d7"} {"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:27.765245762Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1915} {"msg":"Activated sub_volume dae107c7-0bd7-420d-a173-2f1902c70ab7","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:27.765253922Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1915} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:27.765262082Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1915,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:27.765605302Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1915,"task":"crutest"} 01/30 Flush 02/30 Write block 464 len 24576 data: 20 18 16 16 17 15 03/30 Write block 25 len 8192 data: 17 14 04/30 Write block 125 len 20480 data: 15 13 10 12 11 05/30 Read block 41 len 24576 06/30 Read block 413 len 20480 07/30 Read block 202 len 16384 08/30 Read block 477 len 4096 09/30 Read block 388 len 40960 10/30 Write block 328 len 20480 data: 14 10 14 18 18 11/30 Write block 314 len 24576 data: 10 8 11 12 13 15 12/30 Write block 343 len 28672 data: 15 17 16 15 18 16 15 13/30 Read block 112 len 28672 14/30 Read block 444 len 32768 15/30 Read block 52 len 32768 16/30 Read block 137 len 8192 17/30 Read block 299 len 24576 18/30 Read block 215 len 16384 19/30 Flush 20/30 Write block 491 len 24576 data: 13 15 17 16 15 14 21/30 Read block 532 len 32768 22/30 Write block 296 len 4096 data: 18 23/30 Read block 352 len 4096 24/30 Read block 222 len 36864 25/30 Read block 491 len 16384 26/30 Read block 381 len 24576 27/30 Read block 197 len 12288 28/30 Write block 469 len 32768 data: 16 14 11 7 7 8 9 13 29/30 Write block 22 len 40960 data: 16 18 18 18 15 16 16 15 17 18 30/30 Write block 263 len 20480 data: 19 18 18 17 18 ---------------------------------------------------------------- Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 6 Done Done Done false 1002 Acked Write 2 Done Done Done false 1003 Acked Write 5 Sent Done Done false 1004 Acked Read 6 Sent Done Done false 1005 Acked Read 5 Sent Done Done false 1006 Acked Read 4 Sent Done Done false 1007 Acked Read 1 Sent Done Done false 1008 Acked Read 10 Sent Done Done false 1009 Acked Write 5 Sent Done Done false 1010 Acked Write 6 Sent Done Done false 1011 Acked Write 7 Sent Done Done false 1012 Acked Read 7 Sent Done Done false 1013 Acked Read 8 Sent Done Done false 1014 Acked Read 8 Sent Done Done false 1015 Acked Read 2 Sent Done Done false 1016 Acked Read 6 Sent Done Done false 1017 Acked Read 4 Sent Done Done false 1018 Acked Flush 0 Sent Done Done false 1019 Acked Write 6 Sent Done Done false 1020 Acked Read 8 Sent Done Done false 1021 Acked Write 1 Sent Done Done false 1022 Acked Read 1 Sent Done Done false 1023 Acked Read 9 Sent Done Done false 1024 Acked Read 4 Sent Done Done false 1025 Acked Read 6 Sent Done Done false 1026 Acked Read 3 Sent Done Done false 1027 Acked Write 8 Sent Sent Sent false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 27 3 3 33 Done 2 26 26 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1000)) Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:26:27.773705785Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1915} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:27.773957205Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1915,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 189 189 189 316 316 316 F F F 1 020-039 186 194 194 313 328 328 T T T <--- 2 040-059 191 191 191 319 319 319 F F F 3 060-079 191 191 191 319 319 319 F F F 4 080-099 192 192 192 320 320 320 F F F 5 100-119 191 191 191 319 319 319 F F F 6 120-139 184 194 194 311 328 328 F F F <--- 7 140-159 190 190 190 318 318 318 F F F 8 160-179 191 191 191 319 319 319 F F F 9 180-199 184 184 184 311 311 311 F F F 10 200-219 189 189 189 316 316 316 F F F 11 220-239 189 189 189 316 316 316 F F F 12 240-259 193 193 193 326 326 326 F F F 13 260-279 185 185 185 312 312 312 F T T <--- 14 280-299 190 190 190 318 318 318 F T T <--- 15 300-319 192 194 194 320 328 328 F F F <--- 16 320-339 192 194 194 324 328 328 F F F <--- 17 340-359 192 194 194 320 328 328 F F F <--- 18 360-379 192 192 192 320 320 320 F F F 19 380-399 190 190 190 317 317 317 F F F 20 400-419 189 189 189 316 316 316 F F F 21 420-439 193 193 193 326 326 326 F F F 22 440-459 193 193 193 326 326 326 F F F 23 460-479 192 194 194 324 328 328 T T T <--- 24 480-499 192 192 192 324 324 324 F T T <--- 25 500-519 192 192 192 320 320 320 F F F 26 520-539 190 190 190 317 317 317 F F F 27 540-559 190 190 190 317 317 317 F F F 28 560-579 192 192 192 325 325 325 F F F 29 580-599 192 192 192 320 320 320 F F F Max gen: 194, Max flush: 328 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 97 Verifying data now No change after verify Loop: 97 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 189 189 189 316 316 316 F F F 1 020-039 195 195 195 329 329 329 F F F 2 040-059 191 191 191 319 319 319 F F F 3 060-079 191 191 191 319 319 319 F F F 4 080-099 192 192 192 320 320 320 F F F 5 100-119 191 191 191 319 319 319 F F F 6 120-139 194 194 194 328 328 328 F F F 7 140-159 190 190 190 318 318 318 F F F 8 160-179 191 191 191 319 319 319 F F F 9 180-199 184 184 184 311 311 311 F F F 10 200-219 189 189 189 316 316 316 F F F 11 220-239 189 189 189 316 316 316 F F F 12 240-259 193 193 193 326 326 326 F F F 13 260-279 195 195 195 329 329 329 F F F 14 280-299 195 195 195 329 329 329 F F F 15 300-319 194 194 194 328 328 328 F F F 16 320-339 194 194 194 328 328 328 F F F 17 340-359 194 194 194 328 328 328 F F F 18 360-379 192 192 192 320 320 320 F F F 19 380-399 190 190 190 317 317 317 F F F 20 400-419 189 189 189 316 316 316 F F F 21 420-439 193 193 193 326 326 326 F F F 22 440-459 193 193 193 326 326 326 F F F 23 460-479 195 195 195 329 329 329 F F F 24 480-499 195 195 195 329 329 329 F F F 25 500-519 192 192 192 320 320 320 F F F 26 520-539 190 190 190 317 317 317 F F F 27 540-559 190 190 190 317 317 317 F F F 28 560-579 192 192 192 325 325 325 F F F 29 580-599 192 192 192 320 320 320 F F F Max gen: 195, Max flush: 329 Begin loop 98 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:32.024198798Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1923,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:26:32.031399359Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1923,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:32.031454779Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1923,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:32.038188699Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1923,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:32.0384586Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1923} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:32.03848673Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1923} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:32.03849792Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1923} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:32.03850986Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1923} {"msg":"Crucible c30430b1-6bf1-4cd3-b4de-d232926f3a89 has session id: b805ab43-e1be-4992-ad5d-b926c3d63a87","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:32.03888104Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1923,"session_id":"b805ab43-e1be-4992-ad5d-b926c3d63a87"} {"msg":"Upstairs opts: Upstairs UUID: c30430b1-6bf1-4cd3-b4de-d232926f3a89, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:32.03891084Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1923,"session_id":"b805ab43-e1be-4992-ad5d-b926c3d63a87"} {"msg":"Crucible stats registered with UUID: c30430b1-6bf1-4cd3-b4de-d232926f3a89","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:32.03892228Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1923,"session_id":"b805ab43-e1be-4992-ad5d-b926c3d63a87"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:26:32.03893164Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1923,"session_id":"b805ab43-e1be-4992-ad5d-b926c3d63a87"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:32.039076281Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1923,"":"io task","client":"0","":"downstairs","session_id":"b805ab43-e1be-4992-ad5d-b926c3d63a87"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:32.039108371Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1923,"":"io task","client":"1","":"downstairs","session_id":"b805ab43-e1be-4992-ad5d-b926c3d63a87"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:32.039119321Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1923,"":"io task","client":"2","":"downstairs","session_id":"b805ab43-e1be-4992-ad5d-b926c3d63a87"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:32.039272221Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1923,"task":"crutest"} {"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:32.039289201Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1923} {"msg":"c30430b1-6bf1-4cd3-b4de-d232926f3a89 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:32.039321111Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1923,"session_id":"b805ab43-e1be-4992-ad5d-b926c3d63a87"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:32.039330691Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1923,"client":"0","":"downstairs","session_id":"b805ab43-e1be-4992-ad5d-b926c3d63a87"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:32.039340471Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1923,"client":"1","":"downstairs","session_id":"b805ab43-e1be-4992-ad5d-b926c3d63a87"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:32.039350251Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1923,"client":"2","":"downstairs","session_id":"b805ab43-e1be-4992-ad5d-b926c3d63a87"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:32.039361871Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1923,"":"io task","client":"2","":"downstairs","session_id":"b805ab43-e1be-4992-ad5d-b926c3d63a87"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:32.039371871Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1923,"":"io task","client":"0","":"downstairs","session_id":"b805ab43-e1be-4992-ad5d-b926c3d63a87"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:32.039381931Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1923,"":"io task","client":"1","":"downstairs","session_id":"b805ab43-e1be-4992-ad5d-b926c3d63a87"} {"msg":"ds_connection connected from Ok(127.0.0.1:37271)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:32.039426781Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1923,"":"io task","client":"0","":"downstairs","session_id":"b805ab43-e1be-4992-ad5d-b926c3d63a87"} {"msg":"ds_connection connected from Ok(127.0.0.1:41521)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:32.039437801Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1923,"":"io task","client":"2","":"downstairs","session_id":"b805ab43-e1be-4992-ad5d-b926c3d63a87"} {"msg":"ds_connection connected from Ok(127.0.0.1:36432)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:32.039448151Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1923,"":"io task","client":"1","":"downstairs","session_id":"b805ab43-e1be-4992-ad5d-b926c3d63a87"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:32.040125212Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1923,"client":"1","":"downstairs","session_id":"b805ab43-e1be-4992-ad5d-b926c3d63a87"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:32.040146712Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1923,"client":"0","":"downstairs","session_id":"b805ab43-e1be-4992-ad5d-b926c3d63a87"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:32.040162402Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1923,"client":"2","":"downstairs","session_id":"b805ab43-e1be-4992-ad5d-b926c3d63a87"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:32.040452373Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1923,"session_id":"b805ab43-e1be-4992-ad5d-b926c3d63a87"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:32.040462743Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1923,"session_id":"b805ab43-e1be-4992-ad5d-b926c3d63a87"} {"msg":"[0]R flush_numbers[0..12]: [316, 329, 319, 319, 320, 319, 328, 318, 319, 311, 316, 316]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:32.040502343Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1923,"":"downstairs","":"downstairs","session_id":"b805ab43-e1be-4992-ad5d-b926c3d63a87"} {"msg":"[0]R generation[0..12]: [189, 195, 191, 191, 192, 191, 194, 190, 191, 184, 189, 189]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:32.040513743Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1923,"":"downstairs","":"downstairs","session_id":"b805ab43-e1be-4992-ad5d-b926c3d63a87"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:32.040524903Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1923,"":"downstairs","":"downstairs","session_id":"b805ab43-e1be-4992-ad5d-b926c3d63a87"} {"msg":"[1]R flush_numbers[0..12]: [316, 329, 319, 319, 320, 319, 328, 318, 319, 311, 316, 316]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:32.040535923Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1923,"":"downstairs","":"downstairs","session_id":"b805ab43-e1be-4992-ad5d-b926c3d63a87"} {"msg":"[1]R generation[0..12]: [189, 195, 191, 191, 192, 191, 194, 190, 191, 184, 189, 189]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:32.040548943Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1923,"":"downstairs","":"downstairs","session_id":"b805ab43-e1be-4992-ad5d-b926c3d63a87"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:32.040560153Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1923,"":"downstairs","":"downstairs","session_id":"b805ab43-e1be-4992-ad5d-b926c3d63a87"} {"msg":"[2]R flush_numbers[0..12]: [316, 329, 319, 319, 320, 319, 328, 318, 319, 311, 316, 316]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:32.040570883Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1923,"":"downstairs","":"downstairs","session_id":"b805ab43-e1be-4992-ad5d-b926c3d63a87"} {"msg":"[2]R generation[0..12]: [189, 195, 191, 191, 192, 191, 194, 190, 191, 184, 189, 189]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:32.040581703Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1923,"":"downstairs","":"downstairs","session_id":"b805ab43-e1be-4992-ad5d-b926c3d63a87"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:32.040592773Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1923,"":"downstairs","":"downstairs","session_id":"b805ab43-e1be-4992-ad5d-b926c3d63a87"} {"msg":"Max found gen is 196","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:32.040602233Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1923,"":"downstairs","":"downstairs","session_id":"b805ab43-e1be-4992-ad5d-b926c3d63a87"} {"msg":"Generation requested: 196 >= found:196","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:32.040612213Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1923,"":"downstairs","":"downstairs","session_id":"b805ab43-e1be-4992-ad5d-b926c3d63a87"} {"msg":"Next flush: 330","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:32.040621683Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1923,"":"downstairs","":"downstairs","session_id":"b805ab43-e1be-4992-ad5d-b926c3d63a87"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:32.040631183Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1923,"":"downstairs","":"downstairs","session_id":"b805ab43-e1be-4992-ad5d-b926c3d63a87"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:32.040640983Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1923,"session_id":"b805ab43-e1be-4992-ad5d-b926c3d63a87"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:32.040649933Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1923,"client":"0","":"downstairs","session_id":"b805ab43-e1be-4992-ad5d-b926c3d63a87"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:32.040659813Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1923,"client":"1","":"downstairs","session_id":"b805ab43-e1be-4992-ad5d-b926c3d63a87"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:32.040669553Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1923,"client":"2","":"downstairs","session_id":"b805ab43-e1be-4992-ad5d-b926c3d63a87"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:32.040679543Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1923,"session_id":"b805ab43-e1be-4992-ad5d-b926c3d63a87"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:32.040688733Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1923,"session_id":"b805ab43-e1be-4992-ad5d-b926c3d63a87"} {"msg":"c30430b1-6bf1-4cd3-b4de-d232926f3a89 is now active with session: b805ab43-e1be-4992-ad5d-b926c3d63a87","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:32.040700063Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1923,"session_id":"b805ab43-e1be-4992-ad5d-b926c3d63a87"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:32.040708813Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1923,"session_id":"b805ab43-e1be-4992-ad5d-b926c3d63a87"} {"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:32.040718093Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1923} {"msg":"Activated sub_volume c30430b1-6bf1-4cd3-b4de-d232926f3a89","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:32.040726183Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1923} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:32.040734313Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1923,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:32.041053344Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1923,"task":"crutest"} 01/30 Flush 02/30 Write block 99 len 24576 data: 9 9 7 8 10 13 03/30 Read block 570 len 36864 04/30 Flush 05/30 Write block 383 len 20480 data: 15 14 16 15 11 06/30 Write block 550 len 36864 data: 13 12 14 8 7 10 12 16 12 07/30 Write block 420 len 4096 data: 16 08/30 Read block 123 len 36864 09/30 Read block 261 len 4096 10/30 Read block 101 len 32768 11/30 Read block 567 len 4096 12/30 Write block 354 len 12288 data: 13 13 12 13/30 Write block 505 len 4096 data: 13 14/30 Read block 354 len 24576 15/30 Write block 94 len 24576 data: 18 17 15 15 15 10 16/30 Read block 209 len 4096 17/30 Read block 145 len 4096 18/30 Read block 340 len 20480 19/30 Read block 510 len 4096 20/30 Write block 396 len 24576 data: 13 11 8 10 9 8 21/30 Read block 146 len 24576 22/30 Write block 433 len 20480 data: 11 10 9 9 11 23/30 Read block 564 len 4096 24/30 Write block 64 len 36864 data: 18 17 16 16 14 16 17 15 12 25/30 Read block 365 len 32768 26/30 Read block 491 len 4096 27/30 Read block 218 len 32768 28/30 Read block 192 len 4096 29/30 Read block 83 len 36864 30/30 Read block 430 len 32768 ---------------------------------------------------------------- Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Flush 0 Sent Done Done false 1001 Acked Write 6 Sent Done Done false 1002 Acked Read 9 Sent Done Done false 1003 Acked Flush 0 Sent Done Done false 1004 Acked Write 5 Sent Done Done false 1005 Acked Write 9 Sent Done Done false 1006 Acked Write 1 Sent Done Done false 1007 Acked Read 9 Sent Done Done false 1008 Acked Read 1 Sent Done Done false 1009 Acked Read 8 Sent Done Done false 1010 Acked Read 1 Sent Done Done false 1011 Acked Write 3 Sent Done Done false 1012 Acked Write 1 Sent Done Done false 1013 Acked Read 6 Sent Done Done false 1014 Acked Write 6 Sent Done Done false 1015 Acked Read 1 Sent Done Done false 1016 Acked Read 1 Sent Done Done false 1017 Acked Read 5 Sent Done Done false 1018 Acked Read 1 Sent Done Done false 1019 Acked Write 6 Sent Done Done false 1020 Acked Read 6 Sent Done Done false 1021 Acked Write 5 Sent Done Done false 1022 Acked Read 1 Sent Done Done false 1023 Acked Write 9 Sent Done Done false 1024 Acked Read 8 Sent Done Done false 1025 Acked Read 1 Sent Done Done false 1026 Acked Read 8 Sent Done Done false 1027 Acked Read 1 Sent Done Done false 1028 Acked Read 9 Sent Done Done false 1029 Acked Read 8 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 0 0 30 Done 0 30 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1003)) Some(JobId(1003)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:26:32.049609227Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1923} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:32.049799427Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1923,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 189 189 189 316 316 316 F F F 1 020-039 195 195 195 329 329 329 F F F 2 040-059 191 191 191 319 319 319 F F F 3 060-079 191 191 191 319 319 319 F T T <--- 4 080-099 192 196 196 320 331 331 F T T <--- 5 100-119 191 196 196 319 331 331 F F F <--- 6 120-139 194 194 194 328 328 328 F F F 7 140-159 190 190 190 318 318 318 F F F 8 160-179 191 191 191 319 319 319 F F F 9 180-199 184 184 184 311 311 311 F F F 10 200-219 189 189 189 316 316 316 F F F 11 220-239 189 189 189 316 316 316 F F F 12 240-259 193 193 193 326 326 326 F F F 13 260-279 195 195 195 329 329 329 F F F 14 280-299 195 195 195 329 329 329 F F F 15 300-319 194 194 194 328 328 328 F F F 16 320-339 194 194 194 328 328 328 F F F 17 340-359 194 194 194 328 328 328 F T T <--- 18 360-379 192 192 192 320 320 320 F F F 19 380-399 190 190 190 317 317 317 F T T <--- 20 400-419 189 189 189 316 316 316 F T T <--- 21 420-439 193 193 193 326 326 326 F T T <--- 22 440-459 193 193 193 326 326 326 F F F 23 460-479 195 195 195 329 329 329 F F F 24 480-499 195 195 195 329 329 329 F F F 25 500-519 192 192 192 320 320 320 F T T <--- 26 520-539 190 190 190 317 317 317 F F F 27 540-559 190 190 190 317 317 317 F T T <--- 28 560-579 192 192 192 325 325 325 F F F 29 580-599 192 192 192 320 320 320 F F F Max gen: 196, Max flush: 331 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 98 Verifying data now No change after verify Loop: 98 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 189 189 189 316 316 316 F F F 1 020-039 195 195 195 329 329 329 F F F 2 040-059 191 191 191 319 319 319 F F F 3 060-079 197 197 197 332 332 332 F F F 4 080-099 197 197 197 332 332 332 F F F 5 100-119 196 196 196 331 331 331 F F F 6 120-139 194 194 194 328 328 328 F F F 7 140-159 190 190 190 318 318 318 F F F 8 160-179 191 191 191 319 319 319 F F F 9 180-199 184 184 184 311 311 311 F F F 10 200-219 189 189 189 316 316 316 F F F 11 220-239 189 189 189 316 316 316 F F F 12 240-259 193 193 193 326 326 326 F F F 13 260-279 195 195 195 329 329 329 F F F 14 280-299 195 195 195 329 329 329 F F F 15 300-319 194 194 194 328 328 328 F F F 16 320-339 194 194 194 328 328 328 F F F 17 340-359 197 197 197 332 332 332 F F F 18 360-379 192 192 192 320 320 320 F F F 19 380-399 197 197 197 332 332 332 F F F 20 400-419 197 197 197 332 332 332 F F F 21 420-439 197 197 197 332 332 332 F F F 22 440-459 193 193 193 326 326 326 F F F 23 460-479 195 195 195 329 329 329 F F F 24 480-499 195 195 195 329 329 329 F F F 25 500-519 197 197 197 332 332 332 F F F 26 520-539 190 190 190 317 317 317 F F F 27 540-559 197 197 197 332 332 332 F F F 28 560-579 192 192 192 325 325 325 F F F 29 580-599 192 192 192 320 320 320 F F F Max gen: 197, Max flush: 332 Begin loop 99 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:36.300355321Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1931,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:36.308821504Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1931,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:36.309132094Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1931} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ea77faa5daff74362de37fd83a67c9fff3058b9b\",\n git_commit_timestamp: \"2026-07-22T22:05:05.000000000Z\",\n git_branch: \"alan/ttr-fix-log-name\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:36.309159134Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1931} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:36.309167834Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1931} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:36.309179694Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1931} {"msg":"Crucible d7db6568-99f2-4c43-999d-e1eba5445c99 has session id: 47032418-510f-4ae3-8f19-72403700f873","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:36.309515305Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1931,"session_id":"47032418-510f-4ae3-8f19-72403700f873"} {"msg":"Upstairs opts: Upstairs UUID: d7db6568-99f2-4c43-999d-e1eba5445c99, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:36.309544265Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1931,"session_id":"47032418-510f-4ae3-8f19-72403700f873"} {"msg":"Crucible stats registered with UUID: d7db6568-99f2-4c43-999d-e1eba5445c99","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:36.309555395Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1931,"session_id":"47032418-510f-4ae3-8f19-72403700f873"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:26:36.309566785Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1931,"session_id":"47032418-510f-4ae3-8f19-72403700f873"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:36.309688425Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1931,"":"io task","client":"0","":"downstairs","session_id":"47032418-510f-4ae3-8f19-72403700f873"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:36.309706185Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1931,"":"io task","client":"1","":"downstairs","session_id":"47032418-510f-4ae3-8f19-72403700f873"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:36.309716355Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1931,"":"io task","client":"2","":"downstairs","session_id":"47032418-510f-4ae3-8f19-72403700f873"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:36.309882585Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1931,"task":"crutest"} {"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:36.309899356Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1931} {"msg":"d7db6568-99f2-4c43-999d-e1eba5445c99 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:36.309938206Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1931,"session_id":"47032418-510f-4ae3-8f19-72403700f873"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:36.309947696Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1931,"client":"0","":"downstairs","session_id":"47032418-510f-4ae3-8f19-72403700f873"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:36.309957466Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1931,"client":"1","":"downstairs","session_id":"47032418-510f-4ae3-8f19-72403700f873"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:36.309967326Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1931,"client":"2","":"downstairs","session_id":"47032418-510f-4ae3-8f19-72403700f873"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:36.309976866Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1931,"":"io task","client":"2","":"downstairs","session_id":"47032418-510f-4ae3-8f19-72403700f873"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:36.309986906Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1931,"":"io task","client":"0","":"downstairs","session_id":"47032418-510f-4ae3-8f19-72403700f873"} {"msg":"ds_connection connected from Ok(127.0.0.1:41025)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:36.310038826Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1931,"":"io task","client":"2","":"downstairs","session_id":"47032418-510f-4ae3-8f19-72403700f873"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:36.310049416Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1931,"":"io task","client":"1","":"downstairs","session_id":"47032418-510f-4ae3-8f19-72403700f873"} {"msg":"ds_connection connected from Ok(127.0.0.1:62818)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:36.310177416Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1931,"":"io task","client":"1","":"downstairs","session_id":"47032418-510f-4ae3-8f19-72403700f873"} {"msg":"ds_connection connected from Ok(127.0.0.1:33102)","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:36.310193146Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1931,"":"io task","client":"0","":"downstairs","session_id":"47032418-510f-4ae3-8f19-72403700f873"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:36.310639237Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1931,"client":"2","":"downstairs","session_id":"47032418-510f-4ae3-8f19-72403700f873"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:36.310718197Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1931,"client":"0","":"downstairs","session_id":"47032418-510f-4ae3-8f19-72403700f873"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:36.310780597Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1931,"client":"1","":"downstairs","session_id":"47032418-510f-4ae3-8f19-72403700f873"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:36.310917277Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1931,"session_id":"47032418-510f-4ae3-8f19-72403700f873"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:36.311022457Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1931,"session_id":"47032418-510f-4ae3-8f19-72403700f873"} {"msg":"[0]R flush_numbers[0..12]: [316, 329, 319, 332, 332, 331, 328, 318, 319, 311, 316, 316]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:36.311072067Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1931,"":"downstairs","":"downstairs","session_id":"47032418-510f-4ae3-8f19-72403700f873"} {"msg":"[0]R generation[0..12]: [189, 195, 191, 197, 197, 196, 194, 190, 191, 184, 189, 189]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:36.311083707Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1931,"":"downstairs","":"downstairs","session_id":"47032418-510f-4ae3-8f19-72403700f873"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:36.311094917Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1931,"":"downstairs","":"downstairs","session_id":"47032418-510f-4ae3-8f19-72403700f873"} {"msg":"[1]R flush_numbers[0..12]: [316, 329, 319, 332, 332, 331, 328, 318, 319, 311, 316, 316]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:36.311105737Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1931,"":"downstairs","":"downstairs","session_id":"47032418-510f-4ae3-8f19-72403700f873"} {"msg":"[1]R generation[0..12]: [189, 195, 191, 197, 197, 196, 194, 190, 191, 184, 189, 189]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:36.311116497Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1931,"":"downstairs","":"downstairs","session_id":"47032418-510f-4ae3-8f19-72403700f873"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:36.311127547Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1931,"":"downstairs","":"downstairs","session_id":"47032418-510f-4ae3-8f19-72403700f873"} {"msg":"[2]R flush_numbers[0..12]: [316, 329, 319, 332, 332, 331, 328, 318, 319, 311, 316, 316]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:36.311138267Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1931,"":"downstairs","":"downstairs","session_id":"47032418-510f-4ae3-8f19-72403700f873"} {"msg":"[2]R generation[0..12]: [189, 195, 191, 197, 197, 196, 194, 190, 191, 184, 189, 189]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:36.311148977Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1931,"":"downstairs","":"downstairs","session_id":"47032418-510f-4ae3-8f19-72403700f873"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:36.311159987Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1931,"":"downstairs","":"downstairs","session_id":"47032418-510f-4ae3-8f19-72403700f873"} {"msg":"Max found gen is 198","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:36.311171707Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1931,"":"downstairs","":"downstairs","session_id":"47032418-510f-4ae3-8f19-72403700f873"} {"msg":"Generation requested: 198 >= found:198","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:36.311181697Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1931,"":"downstairs","":"downstairs","session_id":"47032418-510f-4ae3-8f19-72403700f873"} {"msg":"Next flush: 333","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:36.311191127Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1931,"":"downstairs","":"downstairs","session_id":"47032418-510f-4ae3-8f19-72403700f873"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:36.311200637Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1931,"":"downstairs","":"downstairs","session_id":"47032418-510f-4ae3-8f19-72403700f873"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:36.311210347Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1931,"session_id":"47032418-510f-4ae3-8f19-72403700f873"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:36.311219167Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1931,"client":"0","":"downstairs","session_id":"47032418-510f-4ae3-8f19-72403700f873"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:36.311228828Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1931,"client":"1","":"downstairs","session_id":"47032418-510f-4ae3-8f19-72403700f873"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:36.311238508Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1931,"client":"2","":"downstairs","session_id":"47032418-510f-4ae3-8f19-72403700f873"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:36.311248428Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1931,"session_id":"47032418-510f-4ae3-8f19-72403700f873"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:36.311257768Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1931,"session_id":"47032418-510f-4ae3-8f19-72403700f873"} {"msg":"d7db6568-99f2-4c43-999d-e1eba5445c99 is now active with session: 47032418-510f-4ae3-8f19-72403700f873","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:36.311267818Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1931,"session_id":"47032418-510f-4ae3-8f19-72403700f873"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:36.311276678Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1931,"session_id":"47032418-510f-4ae3-8f19-72403700f873"} {"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:36.311285998Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1931} {"msg":"Activated sub_volume d7db6568-99f2-4c43-999d-e1eba5445c99","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:36.311294198Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1931} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:36.311302348Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1931,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:36.311628068Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1931,"task":"crutest"} 01/30 Write block 318 len 16384 data: 14 16 16 16 02/30 Read block 160 len 28672 03/30 Read block 405 len 12288 04/30 Read block 373 len 12288 05/30 Read block 448 len 24576 06/30 Flush 07/30 Write block 369 len 32768 data: 12 11 13 15 12 14 15 17 08/30 Write block 223 len 40960 data: 14 11 13 13 14 14 10 10 9 8 09/30 Read block 414 len 40960 10/30 Write block 65 len 12288 data: 18 17 17 11/30 Read block 256 len 36864 12/30 Read block 147 len 32768 13/30 Flush 14/30 Write block 206 len 40960 data: 14 14 15 16 16 17 15 13 13 17 15/30 Read block 67 len 40960 16/30 Read block 252 len 20480 17/30 Read block 501 len 8192 18/30 Write block 251 len 28672 data: 20 15 12 10 12 10 10 19/30 Read block 318 len 8192 20/30 Write block 389 len 32768 data: 13 13 13 14 12 15 14 14 21/30 Read block 159 len 36864 22/30 Write block 38 len 20480 data: 10 12 10 13 13 23/30 Read block 111 len 40960 24/30 Read block 539 len 24576 25/30 Flush 26/30 Write block 63 len 40960 data: 21 19 19 18 18 15 17 18 16 13 27/30 Read block 64 len 8192 28/30 Read block 327 len 24576 29/30 Write block 158 len 36864 data: 20 17 16 15 9 8 9 9 11 30/30 Read block 26 len 4096 ---------------------------------------------------------------- Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 8 Done Done Done false 1007 Acked Write 10 Done Done Done false 1008 Acked Read 10 Done Done Done false 1009 Acked Write 3 Done Done Done false 1010 Acked Read 9 Done Done Done false 1011 Acked Read 8 Done Done Done false 1012 Acked Flush 0 Done Sent Done false 1013 Acked Write 10 Done Sent Done false 1014 Acked Read 10 Done Sent Done false 1015 Acked Read 5 Done Sent Done false 1016 Acked Read 2 Done Sent Done false 1017 Acked Write 7 Done Sent Done false 1018 Acked Read 2 Done Sent Done false 1019 Acked Write 8 Done Sent Done false 1020 Acked Read 9 Done Sent Done false 1021 Acked Write 5 Done Sent Done false 1022 Acked Read 10 Done Sent Done false 1023 Acked Read 6 Done Sent Done false 1024 Acked Flush 0 Done Sent Done false 1025 Acked Write 10 Done Sent Done false 1026 Acked Read 2 Done Sent Done false 1027 Acked Read 6 Done Sent Done false 1028 Acked Write 9 Done Sent Done false 1029 Acked Read 1 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 18 0 18 Done 24 6 24 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1005)) Some(JobId(1024)) Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-22T22:26:36.321910324Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1931} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-22T22:26:36.322106984Z","hostname":"w-01KY5YGK9V792E6WY6KV93YHRV","pid":1931,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 189 189 189 316 316 316 F F F 1 020-039 198 195 198 335 329 335 F F F <--- 2 040-059 198 191 198 335 319 335 F F F <--- 3 060-079 198 197 198 334 332 334 T T T <--- 4 080-099 197 197 197 332 332 332 F F F 5 100-119 196 196 196 331 331 331 F F F 6 120-139 194 194 194 328 328 328 F F F 7 140-159 190 190 190 318 318 318 T F T <--- 8 160-179 191 191 191 319 319 319 T F T <--- 9 180-199 184 184 184 311 311 311 F F F 10 200-219 198 189 198 335 316 335 F F F <--- 11 220-239 198 189 198 334 316 334 F T F <--- 12 240-259 198 193 198 335 326 335 F F F <--- 13 260-279 195 195 195 329 329 329 F F F 14 280-299 195 195 195 329 329 329 F F F 15 300-319 198 198 198 333 333 333 F F F 16 320-339 198 198 198 333 333 333 F F F 17 340-359 197 197 197 332 332 332 F F F 18 360-379 198 192 198 334 320 334 F T F <--- 19 380-399 198 197 198 335 332 335 F F F <--- 20 400-419 197 197 197 332 332 332 F F F 21 420-439 197 197 197 332 332 332 F F F 22 440-459 193 193 193 326 326 326 F F F 23 460-479 195 195 195 329 329 329 F F F 24 480-499 195 195 195 329 329 329 F F F 25 500-519 197 197 197 332 332 332 F F F 26 520-539 190 190 190 317 317 317 F F F 27 540-559 197 197 197 332 332 332 F F F 28 560-579 192 192 192 325 325 325 F F F 29 580-599 192 192 192 320 320 320 F F F Max gen: 198, Max flush: 335 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 99 Verifying data now No change after verify Loop: 99 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 189 189 189 316 316 316 F F F 1 020-039 198 198 198 335 335 335 F F F 2 040-059 198 198 198 335 335 335 F F F 3 060-079 199 199 199 336 336 336 F F F 4 080-099 197 197 197 332 332 332 F F F 5 100-119 196 196 196 331 331 331 F F F 6 120-139 194 194 194 328 328 328 F F F 7 140-159 199 199 199 336 336 336 F F F 8 160-179 199 199 199 336 336 336 F F F 9 180-199 184 184 184 311 311 311 F F F 10 200-219 198 198 198 335 335 335 F F F 11 220-239 198 198 198 334 334 334 F F F 12 240-259 198 198 198 335 335 335 F F F 13 260-279 195 195 195 329 329 329 F F F 14 280-299 195 195 195 329 329 329 F F F 15 300-319 198 198 198 333 333 333 F F F 16 320-339 198 198 198 333 333 333 F F F 17 340-359 197 197 197 332 332 332 F F F 18 360-379 198 198 198 334 334 334 F F F 19 380-399 198 198 198 335 335 335 F F F 20 400-419 197 197 197 332 332 332 F F F 21 420-439 197 197 197 332 332 332 F F F 22 440-459 193 193 193 326 326 326 F F F 23 460-479 195 195 195 329 329 329 F F F 24 480-499 195 195 195 329 329 329 F F F 25 500-519 197 197 197 332 332 332 F F F 26 520-539 190 190 190 317 317 317 F F F 27 540-559 197 197 197 332 332 332 F F F 28 560-579 192 192 192 325 325 325 F F F 29 580-599 192 192 192 320 320 320 F F F Max gen: 199, Max flush: 336 7:28 Test duration Test completed real 7:27.905669280 user 25.367607779 sys 14.415412117 trap 0.019896780 tflt 0.066612880 dflt 0.067472292 kflt 0.003808071 lock 11:43:11.005615913 slp 35:46.460034420 lat 2.353419164 stop 0.019589750