Turn off color for downstairs dump Removing existing region "/var/tmp/test_repair" Creating region directory at: /var/tmp/test_repair Creating dsc directory at: /tmp/test_repair/test_repair_dsc Downstairs region 0 created at /var/tmp/test_repair/8810 in 0.2004446 Downstairs region 1 created at /var/tmp/test_repair/8820 in 0.012853514 Downstairs region 2 created at /var/tmp/test_repair/8830 in 0.012257091 Created 3 regions /var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1134 1135 1136 Dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:25.810139621Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:25.825941349Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:25.828607875Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:25.828676415Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:25.828685764Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:25.828697672Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139} {"msg":"Crucible decc5ea0-1cf7-4d66-91d4-2002a373a474 has session id: 50f43781-03ba-46f4-9cd0-d355d9278da8","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:25.829684197Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139,"session_id":"50f43781-03ba-46f4-9cd0-d355d9278da8"} {"msg":"Upstairs opts: Upstairs UUID: decc5ea0-1cf7-4d66-91d4-2002a373a474, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:25.829985572Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139,"session_id":"50f43781-03ba-46f4-9cd0-d355d9278da8"} {"msg":"Crucible stats registered with UUID: decc5ea0-1cf7-4d66-91d4-2002a373a474","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:25.829996391Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139,"session_id":"50f43781-03ba-46f4-9cd0-d355d9278da8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:53:25.830006439Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139,"session_id":"50f43781-03ba-46f4-9cd0-d355d9278da8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:25.830657313Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139,"":"io task","client":"2","":"downstairs","session_id":"50f43781-03ba-46f4-9cd0-d355d9278da8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:25.830915265Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139,"":"io task","client":"0","":"downstairs","session_id":"50f43781-03ba-46f4-9cd0-d355d9278da8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:25.830930483Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139,"":"io task","client":"1","":"downstairs","session_id":"50f43781-03ba-46f4-9cd0-d355d9278da8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:25.831236288Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139,"task":"crutest"} {"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:25.8312868Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139} {"msg":"decc5ea0-1cf7-4d66-91d4-2002a373a474 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:25.831295779Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139,"session_id":"50f43781-03ba-46f4-9cd0-d355d9278da8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:25.831304718Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139,"client":"0","":"downstairs","session_id":"50f43781-03ba-46f4-9cd0-d355d9278da8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:25.831314356Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139,"client":"1","":"downstairs","session_id":"50f43781-03ba-46f4-9cd0-d355d9278da8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:25.831323925Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139,"client":"2","":"downstairs","session_id":"50f43781-03ba-46f4-9cd0-d355d9278da8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:25.831333663Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139,"":"io task","client":"2","":"downstairs","session_id":"50f43781-03ba-46f4-9cd0-d355d9278da8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:25.831343622Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139,"":"io task","client":"0","":"downstairs","session_id":"50f43781-03ba-46f4-9cd0-d355d9278da8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:25.8313589Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139,"":"io task","client":"1","":"downstairs","session_id":"50f43781-03ba-46f4-9cd0-d355d9278da8"} {"msg":"ds_connection connected from Ok(127.0.0.1:48650)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:25.83142256Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139,"":"io task","client":"2","":"downstairs","session_id":"50f43781-03ba-46f4-9cd0-d355d9278da8"} {"msg":"ds_connection connected from Ok(127.0.0.1:47423)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:25.831439458Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139,"":"io task","client":"1","":"downstairs","session_id":"50f43781-03ba-46f4-9cd0-d355d9278da8"} {"msg":"ds_connection connected from Ok(127.0.0.1:41523)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:25.831450446Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139,"":"io task","client":"0","":"downstairs","session_id":"50f43781-03ba-46f4-9cd0-d355d9278da8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:25.833578072Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139,"client":"1","":"downstairs","session_id":"50f43781-03ba-46f4-9cd0-d355d9278da8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:25.833601309Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139,"client":"2","":"downstairs","session_id":"50f43781-03ba-46f4-9cd0-d355d9278da8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:25.833640933Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139,"client":"0","":"downstairs","session_id":"50f43781-03ba-46f4-9cd0-d355d9278da8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:25.834465081Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139,"session_id":"50f43781-03ba-46f4-9cd0-d355d9278da8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:25.834501776Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139,"session_id":"50f43781-03ba-46f4-9cd0-d355d9278da8"} {"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:25.834626388Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139,"":"downstairs","":"downstairs","session_id":"50f43781-03ba-46f4-9cd0-d355d9278da8"} {"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:25.834637546Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139,"":"downstairs","":"downstairs","session_id":"50f43781-03ba-46f4-9cd0-d355d9278da8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:25.834648994Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139,"":"downstairs","":"downstairs","session_id":"50f43781-03ba-46f4-9cd0-d355d9278da8"} {"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:25.834659213Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139,"":"downstairs","":"downstairs","session_id":"50f43781-03ba-46f4-9cd0-d355d9278da8"} {"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:25.834669401Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139,"":"downstairs","":"downstairs","session_id":"50f43781-03ba-46f4-9cd0-d355d9278da8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:25.83468051Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139,"":"downstairs","":"downstairs","session_id":"50f43781-03ba-46f4-9cd0-d355d9278da8"} {"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:25.834691178Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139,"":"downstairs","":"downstairs","session_id":"50f43781-03ba-46f4-9cd0-d355d9278da8"} {"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:25.834701576Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139,"":"downstairs","":"downstairs","session_id":"50f43781-03ba-46f4-9cd0-d355d9278da8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:25.834712435Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139,"":"downstairs","":"downstairs","session_id":"50f43781-03ba-46f4-9cd0-d355d9278da8"} {"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:25.834721943Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139,"":"downstairs","":"downstairs","session_id":"50f43781-03ba-46f4-9cd0-d355d9278da8"} {"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:25.834731502Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139,"":"downstairs","":"downstairs","session_id":"50f43781-03ba-46f4-9cd0-d355d9278da8"} {"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:25.834740821Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139,"":"downstairs","":"downstairs","session_id":"50f43781-03ba-46f4-9cd0-d355d9278da8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:25.834751929Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139,"":"downstairs","":"downstairs","session_id":"50f43781-03ba-46f4-9cd0-d355d9278da8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:25.834761468Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139,"session_id":"50f43781-03ba-46f4-9cd0-d355d9278da8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:25.834770166Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139,"client":"0","":"downstairs","session_id":"50f43781-03ba-46f4-9cd0-d355d9278da8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:25.834779915Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139,"client":"1","":"downstairs","session_id":"50f43781-03ba-46f4-9cd0-d355d9278da8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:25.834789523Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139,"client":"2","":"downstairs","session_id":"50f43781-03ba-46f4-9cd0-d355d9278da8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:25.834799342Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139,"session_id":"50f43781-03ba-46f4-9cd0-d355d9278da8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:25.834808291Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139,"session_id":"50f43781-03ba-46f4-9cd0-d355d9278da8"} {"msg":"decc5ea0-1cf7-4d66-91d4-2002a373a474 is now active with session: 50f43781-03ba-46f4-9cd0-d355d9278da8","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:25.834818099Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139,"session_id":"50f43781-03ba-46f4-9cd0-d355d9278da8"} {"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:25.834827248Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:25.834834717Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139,"session_id":"50f43781-03ba-46f4-9cd0-d355d9278da8"} {"msg":"Activated sub_volume decc5ea0-1cf7-4d66-91d4-2002a373a474","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:25.835263174Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:25.835307377Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Fill test","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:25.835686011Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139,"task":"crutest"} Read and Verify all blocks (0..600 range:false) {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30---------------------------------------------------------------- ,"time":"2026-07-28T18:53:25.876826421Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139,"task":"crutest"} {"msg":"CLIENT: Tests done. All submitted work has been ACK'd","v":0,"name":" Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:3 crucible"States:,"level": Active30 Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY ,"time":"2026-07-28T18:53:25.876865006Z","hostname":" 1004 Acked Read 88w-01KYN136ZFZC5JJKNGKY32PPFH", Done" Done Done falsepid ":1139 1005 Acked Read 256," Done Donetask Done" false: " 1006 Acked Read 256crutest" Done Done} Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1003)) Some(JobId(1003)) Some(JobId(1003)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1006 1005 1004 1003 1002 {"msg":"CLIENT: Up:0 ds:3 act:3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:25.876927037Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139,"task":"crutest"} ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1007 1006 1005 1004 1003 {"msg":"CLIENT: Up:0 ds:0 act:3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:29.878715081Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139,"task":"crutest"} {"msg":"CLIENT: All crucible jobs finished, exiting program","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:29.878763704Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139,"task":"crutest"} {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:53:29.878773832Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139} {"msg":"client stopping due to Err(RecvError(()))","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:29.878818186Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139,"":"io task","client":"0","":"downstairs","session_id":"50f43781-03ba-46f4-9cd0-d355d9278da8"} {"msg":"client request queue closed unexpectedly; is the program exiting?","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:53:29.878834443Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139,"":"io task","client":"1","":"downstairs","session_id":"50f43781-03ba-46f4-9cd0-d355d9278da8"} {"msg":"client task is sending Done(QueueClosed)","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:53:29.878850071Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139,"":"io task","client":"0","":"downstairs","session_id":"50f43781-03ba-46f4-9cd0-d355d9278da8"} {"msg":"client task is sending Done(QueueClosed)","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:53:29.878860849Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139,"":"io task","client":"1","":"downstairs","session_id":"50f43781-03ba-46f4-9cd0-d355d9278da8"} {"msg":"client task could not reply to main task; shutting down?","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:53:29.878871618Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139,"":"io task","client":"0","":"downstairs","session_id":"50f43781-03ba-46f4-9cd0-d355d9278da8"} {"msg":"client task could not reply to main task; shutting down?","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:53:29.878882386Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139,"":"io task","client":"1","":"downstairs","session_id":"50f43781-03ba-46f4-9cd0-d355d9278da8"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:29.878892115Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139,"":"io task","client":"1","":"downstairs","session_id":"50f43781-03ba-46f4-9cd0-d355d9278da8"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:29.878902053Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1139,"":"io task","client":"0","":"downstairs","session_id":"50f43781-03ba-46f4-9cd0-d355d9278da8"} Begin loop 1 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:29.901470443Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1141,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:29.909608823Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1141,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:29.909893751Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1141} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:29.909918737Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1141} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:29.909927546Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1141} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:29.909939104Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1141} {"msg":"Crucible 353f5edb-8f68-4d58-8a0f-e7252619fdbc has session id: f05da181-8b48-4772-9bba-ced28da3ed34","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:29.910301891Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1141,"session_id":"f05da181-8b48-4772-9bba-ced28da3ed34"} {"msg":"Upstairs opts: Upstairs UUID: 353f5edb-8f68-4d58-8a0f-e7252619fdbc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:29.910346774Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1141,"session_id":"f05da181-8b48-4772-9bba-ced28da3ed34"} {"msg":"Crucible stats registered with UUID: 353f5edb-8f68-4d58-8a0f-e7252619fdbc","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:29.910357832Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1141,"session_id":"f05da181-8b48-4772-9bba-ced28da3ed34"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:53:29.910367071Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1141,"session_id":"f05da181-8b48-4772-9bba-ced28da3ed34"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:29.91050534Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1141,"":"io task","client":"0","":"downstairs","session_id":"f05da181-8b48-4772-9bba-ced28da3ed34"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:29.910521648Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1141,"":"io task","client":"2","":"downstairs","session_id":"f05da181-8b48-4772-9bba-ced28da3ed34"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:29.910531977Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1141,"":"io task","client":"1","":"downstairs","session_id":"f05da181-8b48-4772-9bba-ced28da3ed34"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:29.910691613Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1141,"task":"crutest"} {"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:29.910706491Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1141} {"msg":"353f5edb-8f68-4d58-8a0f-e7252619fdbc active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:29.910741466Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1141,"session_id":"f05da181-8b48-4772-9bba-ced28da3ed34"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:29.910751154Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1141,"client":"0","":"downstairs","session_id":"f05da181-8b48-4772-9bba-ced28da3ed34"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:29.910763292Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1141,"client":"1","":"downstairs","session_id":"f05da181-8b48-4772-9bba-ced28da3ed34"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:29.910773041Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1141,"client":"2","":"downstairs","session_id":"f05da181-8b48-4772-9bba-ced28da3ed34"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:29.91078293Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1141,"":"io task","client":"2","":"downstairs","session_id":"f05da181-8b48-4772-9bba-ced28da3ed34"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:29.910793228Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1141,"":"io task","client":"0","":"downstairs","session_id":"f05da181-8b48-4772-9bba-ced28da3ed34"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:29.910803337Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1141,"":"io task","client":"1","":"downstairs","session_id":"f05da181-8b48-4772-9bba-ced28da3ed34"} {"msg":"ds_connection connected from Ok(127.0.0.1:44352)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:29.910837751Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1141,"":"io task","client":"2","":"downstairs","session_id":"f05da181-8b48-4772-9bba-ced28da3ed34"} {"msg":"ds_connection connected from Ok(127.0.0.1:62683)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:29.910861608Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1141,"":"io task","client":"1","":"downstairs","session_id":"f05da181-8b48-4772-9bba-ced28da3ed34"} {"msg":"ds_connection connected from Ok(127.0.0.1:36426)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:29.910964473Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1141,"":"io task","client":"0","":"downstairs","session_id":"f05da181-8b48-4772-9bba-ced28da3ed34"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:29.911509662Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1141,"client":"0","":"downstairs","session_id":"f05da181-8b48-4772-9bba-ced28da3ed34"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:29.91152619Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1141,"client":"1","":"downstairs","session_id":"f05da181-8b48-4772-9bba-ced28da3ed34"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:29.91179449Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1141,"client":"2","":"downstairs","session_id":"f05da181-8b48-4772-9bba-ced28da3ed34"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:29.911867729Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1141,"session_id":"f05da181-8b48-4772-9bba-ced28da3ed34"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:29.911883057Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1141,"session_id":"f05da181-8b48-4772-9bba-ced28da3ed34"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:29.912079508Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1141,"":"downstairs","":"downstairs","session_id":"f05da181-8b48-4772-9bba-ced28da3ed34"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:29.912090647Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1141,"":"downstairs","":"downstairs","session_id":"f05da181-8b48-4772-9bba-ced28da3ed34"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:29.912104545Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1141,"":"downstairs","":"downstairs","session_id":"f05da181-8b48-4772-9bba-ced28da3ed34"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:29.912114813Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1141,"":"downstairs","":"downstairs","session_id":"f05da181-8b48-4772-9bba-ced28da3ed34"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:29.912125112Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1141,"":"downstairs","":"downstairs","session_id":"f05da181-8b48-4772-9bba-ced28da3ed34"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:29.91213599Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1141,"":"downstairs","":"downstairs","session_id":"f05da181-8b48-4772-9bba-ced28da3ed34"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:29.912146248Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1141,"":"downstairs","":"downstairs","session_id":"f05da181-8b48-4772-9bba-ced28da3ed34"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:29.912156457Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1141,"":"downstairs","":"downstairs","session_id":"f05da181-8b48-4772-9bba-ced28da3ed34"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:29.912167255Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1141,"":"downstairs","":"downstairs","session_id":"f05da181-8b48-4772-9bba-ced28da3ed34"} {"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:29.912176864Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1141,"":"downstairs","":"downstairs","session_id":"f05da181-8b48-4772-9bba-ced28da3ed34"} {"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:29.912186392Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1141,"":"downstairs","":"downstairs","session_id":"f05da181-8b48-4772-9bba-ced28da3ed34"} {"msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:29.912195891Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1141,"":"downstairs","":"downstairs","session_id":"f05da181-8b48-4772-9bba-ced28da3ed34"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:29.91220507Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1141,"":"downstairs","":"downstairs","session_id":"f05da181-8b48-4772-9bba-ced28da3ed34"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:29.912214768Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1141,"session_id":"f05da181-8b48-4772-9bba-ced28da3ed34"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:29.912223457Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1141,"client":"0","":"downstairs","session_id":"f05da181-8b48-4772-9bba-ced28da3ed34"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:29.912233516Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1141,"client":"1","":"downstairs","session_id":"f05da181-8b48-4772-9bba-ced28da3ed34"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:29.912243424Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1141,"client":"2","":"downstairs","session_id":"f05da181-8b48-4772-9bba-ced28da3ed34"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:29.912254742Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1141,"session_id":"f05da181-8b48-4772-9bba-ced28da3ed34"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:29.912263831Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1141,"session_id":"f05da181-8b48-4772-9bba-ced28da3ed34"} {"msg":"353f5edb-8f68-4d58-8a0f-e7252619fdbc is now active with session: f05da181-8b48-4772-9bba-ced28da3ed34","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:29.91227381Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1141,"session_id":"f05da181-8b48-4772-9bba-ced28da3ed34"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:29.912282548Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1141,"session_id":"f05da181-8b48-4772-9bba-ced28da3ed34"} {"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:29.912291337Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1141} {"msg":"Activated sub_volume 353f5edb-8f68-4d58-8a0f-e7252619fdbc","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:29.912299366Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1141} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:29.912307455Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1141,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:29.91281295Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1141,"task":"crutest"} 01/30 Write block 266 len 8192 data: 2 2 02/30 Write block 271 len 16384 data: 2 2 2 2 03/30 Write block 339 len 4096 data: 2 04/30 Read block 233 len 36864 05/30 Write block 263 len 20480 data: 2 2 2 3 3 06/30 Read block 86 len 32768 07/30 Read block 161 len 20480 08/30 Read block 404 len 20480 09/30 Read block 10 len 20480 10/30 Write block 450 len 12288 data: 2 2 2 11/30 Flush 12/30 Write block 386 len 36864 data: 2 2 2 2 2 2 2 2 2 13/30 Write block 209 len 4096 data: 2 14/30 Write block 518 len 20480 data: 2 2 2 2 2 15/30 Write block 552 len 28672 data: 2 2 2 2 2 2 2 16/30 Write block 510 len 4096 data: 2 17/30 Read block 133 len 32768 18/30 Write block 125 len 12288 data: 2 2 2 19/30 Read block 263 len 4096 20/30 Read block 559 len 4096 21/30 Read block 383 len 32768 22/30 Read block 552 len 8192 23/30 Read block 57 len 28672 24/30 Write block 390 len 24576 data: 3 3 3 3 3 2 25/30 Write block 333 len 32768 data: 2 2 2 2 2 2 3 2 26/30 Write block 462 len 40960 data: 2 2 2 2 2 2 2 2 2 2 27/30 Read block 408 len 24576 28/30 Read block 538 len 36864 29/30 Write block 318 len 40960 data: 2 2 2 2 2 2 2 2 2 2 30/30 Read block 153 len 32768 ---------------------------------------------------------------- Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Sent false 1001 Acked Write 4 Done Done Sent false 1002 Acked Write 1 Done Done Sent false 1003 Acked Read 9 Done Done Sent false 1004 Acked Write 5 Done Done Sent false 1005 Acked Read 8 Done Done Sent false 1006 Acked Read 5 Done Done Sent false 1007 Acked Read 5 Done Done Sent false 1008 Acked Read 5 Done Done Sent false 1009 Acked Write 3 Done Done Sent false 1010 Acked Flush 0 Done Done Sent false 1011 Acked Write 9 Done Done Sent false 1012 Acked Write 1 Done Done Sent false 1013 Acked Write 5 Done Done Sent false 1014 Acked Write 7 Done Done Sent false 1015 Acked Write 1 Done Done Sent false 1016 Acked Read 8 Done Done Sent false 1017 Acked Write 3 Done Done Sent false 1018 Acked Read 1 Done Done Sent false 1019 Acked Read 1 Done Done Sent false 1020 Acked Read 8 Done Done Sent false 1021 Acked Read 2 Done Done Sent false 1022 Acked Read 7 Done Done Sent false 1023 Acked Write 6 Done Done Sent false 1024 Acked Write 8 Done Done Sent false 1025 Acked Write 10 Done Done Sent false 1026 Acked Read 6 Done Done Sent false 1027 Acked Read 9 Done Done Sent false 1028 Acked Write 10 Done Done Sent false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 30 31 Done 30 29 0 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1010)) Some(JobId(1010)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:53:29.920884979Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1141} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:29.921058374Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1141,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 1 1 1 1 1 1 F F F 3 060-079 1 1 1 1 1 1 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 1 1 1 1 1 1 T T F <--- 7 140-159 1 1 1 1 1 1 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 T T F <--- 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 2 2 1 2 2 1 F F F <--- 14 280-299 1 1 1 1 1 1 F F F 15 300-319 1 1 1 1 1 1 T T F <--- 16 320-339 2 2 1 2 2 1 T T F <--- 17 340-359 1 1 1 1 1 1 T T F <--- 18 360-379 1 1 1 1 1 1 F F F 19 380-399 1 1 1 1 1 1 T T F <--- 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 2 2 1 2 2 1 F F F <--- 23 460-479 1 1 1 1 1 1 T T F <--- 24 480-499 1 1 1 1 1 1 F F F 25 500-519 1 1 1 1 1 1 T T F <--- 26 520-539 1 1 1 1 1 1 T T F <--- 27 540-559 1 1 1 1 1 1 T T F <--- 28 560-579 1 1 1 1 1 1 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 2, Max flush: 2 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 1 Verifying data now No change after verify Loop: 1 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 1 1 1 1 1 1 F F F 3 060-079 1 1 1 1 1 1 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 3 3 3 3 3 3 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 3 3 3 3 3 3 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 2 2 2 2 2 2 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 3 3 3 3 3 3 F F F 16 320-339 3 3 3 3 3 3 F F F 17 340-359 3 3 3 3 3 3 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 3 3 3 3 3 3 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 2 2 2 2 2 2 F F F 23 460-479 3 3 3 3 3 3 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 3 3 3 3 3 3 F F F 26 520-539 3 3 3 3 3 3 F F F 27 540-559 3 3 3 3 3 3 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 3, Max flush: 3 Begin loop 2 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:34.299187673Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1149,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:53:34.306499845Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1149,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:34.306555356Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1149,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:34.313474666Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1149,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:34.313733937Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1149} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:34.313756804Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1149} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:34.313765393Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1149} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:34.313777131Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1149} {"msg":"Crucible f8d656e2-7737-4c98-8f2a-923c5711e7e9 has session id: ae6a11ac-1088-48ae-a90a-db95e20aaeb6","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:34.314135538Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1149,"session_id":"ae6a11ac-1088-48ae-a90a-db95e20aaeb6"} {"msg":"Upstairs opts: Upstairs UUID: f8d656e2-7737-4c98-8f2a-923c5711e7e9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:34.314164574Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1149,"session_id":"ae6a11ac-1088-48ae-a90a-db95e20aaeb6"} {"msg":"Crucible stats registered with UUID: f8d656e2-7737-4c98-8f2a-923c5711e7e9","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:34.314175462Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1149,"session_id":"ae6a11ac-1088-48ae-a90a-db95e20aaeb6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:53:34.314184751Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1149,"session_id":"ae6a11ac-1088-48ae-a90a-db95e20aaeb6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:34.314309722Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1149,"":"io task","client":"1","":"downstairs","session_id":"ae6a11ac-1088-48ae-a90a-db95e20aaeb6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:34.314340418Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1149,"":"io task","client":"0","":"downstairs","session_id":"ae6a11ac-1088-48ae-a90a-db95e20aaeb6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:34.314351776Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1149,"":"io task","client":"2","":"downstairs","session_id":"ae6a11ac-1088-48ae-a90a-db95e20aaeb6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:34.31446323Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1149,"task":"crutest"} {"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:34.314480987Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1149} {"msg":"f8d656e2-7737-4c98-8f2a-923c5711e7e9 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:34.314496955Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1149,"session_id":"ae6a11ac-1088-48ae-a90a-db95e20aaeb6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:34.314505853Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1149,"client":"0","":"downstairs","session_id":"ae6a11ac-1088-48ae-a90a-db95e20aaeb6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:34.314515582Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1149,"client":"1","":"downstairs","session_id":"ae6a11ac-1088-48ae-a90a-db95e20aaeb6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:34.31452531Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1149,"client":"2","":"downstairs","session_id":"ae6a11ac-1088-48ae-a90a-db95e20aaeb6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:34.314535119Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1149,"":"io task","client":"2","":"downstairs","session_id":"ae6a11ac-1088-48ae-a90a-db95e20aaeb6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:34.314545338Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1149,"":"io task","client":"0","":"downstairs","session_id":"ae6a11ac-1088-48ae-a90a-db95e20aaeb6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:34.314557936Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1149,"":"io task","client":"1","":"downstairs","session_id":"ae6a11ac-1088-48ae-a90a-db95e20aaeb6"} {"msg":"ds_connection connected from Ok(127.0.0.1:63314)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:34.314620726Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1149,"":"io task","client":"2","":"downstairs","session_id":"ae6a11ac-1088-48ae-a90a-db95e20aaeb6"} {"msg":"ds_connection connected from Ok(127.0.0.1:57703)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:34.314646273Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1149,"":"io task","client":"1","":"downstairs","session_id":"ae6a11ac-1088-48ae-a90a-db95e20aaeb6"} {"msg":"ds_connection connected from Ok(127.0.0.1:42573)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:34.314657571Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1149,"":"io task","client":"0","":"downstairs","session_id":"ae6a11ac-1088-48ae-a90a-db95e20aaeb6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:34.315314634Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1149,"client":"1","":"downstairs","session_id":"ae6a11ac-1088-48ae-a90a-db95e20aaeb6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:34.315328262Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1149,"client":"2","":"downstairs","session_id":"ae6a11ac-1088-48ae-a90a-db95e20aaeb6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:34.315374985Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1149,"client":"0","":"downstairs","session_id":"ae6a11ac-1088-48ae-a90a-db95e20aaeb6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:34.315642836Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1149,"session_id":"ae6a11ac-1088-48ae-a90a-db95e20aaeb6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:34.315652504Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1149,"session_id":"ae6a11ac-1088-48ae-a90a-db95e20aaeb6"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 3, 1, 1, 1, 3, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:34.315754399Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1149,"":"downstairs","":"downstairs","session_id":"ae6a11ac-1088-48ae-a90a-db95e20aaeb6"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 3, 1, 1, 1, 3, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:34.315765238Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1149,"":"downstairs","":"downstairs","session_id":"ae6a11ac-1088-48ae-a90a-db95e20aaeb6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:34.315776246Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1149,"":"downstairs","":"downstairs","session_id":"ae6a11ac-1088-48ae-a90a-db95e20aaeb6"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 3, 1, 1, 1, 3, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:34.315786314Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1149,"":"downstairs","":"downstairs","session_id":"ae6a11ac-1088-48ae-a90a-db95e20aaeb6"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 3, 1, 1, 1, 3, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:34.315796483Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1149,"":"downstairs","":"downstairs","session_id":"ae6a11ac-1088-48ae-a90a-db95e20aaeb6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:34.315807341Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1149,"":"downstairs","":"downstairs","session_id":"ae6a11ac-1088-48ae-a90a-db95e20aaeb6"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 3, 1, 1, 1, 3, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:34.315819959Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1149,"":"downstairs","":"downstairs","session_id":"ae6a11ac-1088-48ae-a90a-db95e20aaeb6"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 3, 1, 1, 1, 3, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:34.315830068Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1149,"":"downstairs","":"downstairs","session_id":"ae6a11ac-1088-48ae-a90a-db95e20aaeb6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:34.315841026Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1149,"":"downstairs","":"downstairs","session_id":"ae6a11ac-1088-48ae-a90a-db95e20aaeb6"} {"msg":"Max found gen is 4","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:34.315850425Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1149,"":"downstairs","":"downstairs","session_id":"ae6a11ac-1088-48ae-a90a-db95e20aaeb6"} {"msg":"Generation requested: 4 >= found:4","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:34.315860204Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1149,"":"downstairs","":"downstairs","session_id":"ae6a11ac-1088-48ae-a90a-db95e20aaeb6"} {"msg":"Next flush: 4","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:34.315869522Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1149,"":"downstairs","":"downstairs","session_id":"ae6a11ac-1088-48ae-a90a-db95e20aaeb6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:34.315878971Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1149,"":"downstairs","":"downstairs","session_id":"ae6a11ac-1088-48ae-a90a-db95e20aaeb6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:34.315888719Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1149,"session_id":"ae6a11ac-1088-48ae-a90a-db95e20aaeb6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:34.315897498Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1149,"client":"0","":"downstairs","session_id":"ae6a11ac-1088-48ae-a90a-db95e20aaeb6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:34.315907337Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1149,"client":"1","":"downstairs","session_id":"ae6a11ac-1088-48ae-a90a-db95e20aaeb6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:34.315916845Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1149,"client":"2","":"downstairs","session_id":"ae6a11ac-1088-48ae-a90a-db95e20aaeb6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:34.315926814Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1149,"session_id":"ae6a11ac-1088-48ae-a90a-db95e20aaeb6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:34.315935892Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1149,"session_id":"ae6a11ac-1088-48ae-a90a-db95e20aaeb6"} {"msg":"f8d656e2-7737-4c98-8f2a-923c5711e7e9 is now active with session: ae6a11ac-1088-48ae-a90a-db95e20aaeb6","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:34.315945851Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1149,"session_id":"ae6a11ac-1088-48ae-a90a-db95e20aaeb6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:34.31595455Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1149,"session_id":"ae6a11ac-1088-48ae-a90a-db95e20aaeb6"} {"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:34.315963958Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1149} {"msg":"Activated sub_volume f8d656e2-7737-4c98-8f2a-923c5711e7e9","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:34.315973987Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1149} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:34.315982036Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1149,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:34.316271543Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1149,"task":"crutest"} 01/30 Write block 513 len 24576 data: 2 2 2 2 2 3 02/30 Flush 03/30 Write block 466 len 28672 data: 3 3 3 3 3 3 2 04/30 Write block 331 len 8192 data: 2 2 05/30 Read block 552 len 28672 06/30 Write block 487 len 24576 data: 2 2 2 2 2 2 07/30 Write block 98 len 32768 data: 2 2 2 2 2 2 2 2 08/30 Write block 197 len 8192 data: 2 2 09/30 Write block 129 len 24576 data: 2 2 2 2 2 2 10/30 Write block 182 len 12288 data: 2 2 2 11/30 Read block 371 len 40960 12/30 Write block 173 len 36864 data: 2 2 2 2 2 2 2 2 2 13/30 Read block 285 len 28672 14/30 Read block 381 len 20480 15/30 Flush 16/30 Write block 561 len 32768 data: 2 2 2 2 2 2 2 2 17/30 Read block 481 len 28672 18/30 Flush 19/30 Flush 20/30 Write block 478 len 28672 data: 2 2 2 2 2 2 2 21/30 Flush 22/30 Write block 88 len 24576 data: 2 2 2 2 2 2 23/30 Read block 483 len 36864 24/30 Write block 344 len 20480 data: 2 2 2 2 2 25/30 Write block 591 len 16384 data: 2 2 2 2 26/30 Read block 539 len 24576 27/30 Read block 314 len 24576 28/30 Read block 307 len 20480 29/30 Write block 469 len 24576 data: 4 4 4 3 2 2 30/30 Read block 262 len 40960 ---------------------------------------------------------------- Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 7 Done Done Done false 1003 Acked Write 2 Done Done Done false 1004 Acked Read 7 Sent Done Done false 1005 Acked Write 6 Sent Done Done false 1006 Acked Write 8 Sent Done Done false 1007 Acked Write 2 Sent Done Done false 1008 Acked Write 6 Sent Done Done false 1009 Acked Write 3 Sent Done Done false 1010 Acked Read 10 Sent Done Done false 1011 Acked Write 9 Sent Done Done false 1012 Acked Read 7 Sent Done Done false 1013 Acked Read 5 Sent Done Done false 1014 Acked Flush 0 Sent Done Done false 1015 Acked Write 8 Sent Done Done false 1016 Acked Read 7 Sent Done Done false 1017 Acked Flush 0 Sent Done Done false 1018 Acked Flush 0 Sent Done Done false 1019 Acked Write 7 Sent Done Done false 1020 Acked Flush 0 Sent Done Done false 1021 Acked Write 6 Sent Done Done false 1022 Acked Read 9 Sent Done Done false 1023 Acked Write 5 Sent Done Done false 1024 Acked Write 4 Sent Done Done false 1025 Acked Read 6 Sent Done Done false 1026 Acked Read 6 Sent Done Done false 1027 Acked Read 5 Sent Done Done false 1028 Acked Write 6 Sent Done Done false 1029 Acked Read 10 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 26 1 0 27 Done 2 27 28 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1001)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:53:34.325622303Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1149} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:34.325798307Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1149,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 1 1 1 1 1 1 F F F 3 060-079 1 1 1 1 1 1 F F F 4 080-099 1 4 4 1 5 5 F T T <--- 5 100-119 1 4 4 1 5 5 F F F <--- 6 120-139 3 4 4 3 5 5 F F F <--- 7 140-159 1 1 1 1 1 1 F F F 8 160-179 1 4 4 1 5 5 F F F <--- 9 180-199 1 4 4 1 5 5 F F F <--- 10 200-219 3 3 3 3 3 3 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 2 2 2 2 2 2 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 3 3 3 3 3 3 F F F 16 320-339 3 4 4 3 5 5 T F F <--- 17 340-359 3 3 3 3 3 3 F T T <--- 18 360-379 1 1 1 1 1 1 F F F 19 380-399 3 3 3 3 3 3 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 2 2 2 2 2 2 F F F 23 460-479 3 4 4 3 8 8 T T T <--- 24 480-499 1 4 4 1 8 8 F F F <--- 25 500-519 4 4 4 4 4 4 F F F 26 520-539 3 3 3 3 3 3 F F F 27 540-559 3 3 3 3 3 3 F F F 28 560-579 1 4 4 1 6 6 F F F <--- 29 580-599 1 1 1 1 1 1 F T T <--- Max gen: 4, Max flush: 8 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 2 Verifying data now No change after verify Loop: 2 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 1 1 1 1 1 1 F F F 3 060-079 1 1 1 1 1 1 F F F 4 080-099 5 5 5 9 9 9 F F F 5 100-119 4 4 4 5 5 5 F F F 6 120-139 4 4 4 5 5 5 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 4 4 4 5 5 5 F F F 9 180-199 4 4 4 5 5 5 F F F 10 200-219 3 3 3 3 3 3 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 2 2 2 2 2 2 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 3 3 3 3 3 3 F F F 16 320-339 4 4 4 5 5 5 F F F 17 340-359 5 5 5 9 9 9 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 3 3 3 3 3 3 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 2 2 2 2 2 2 F F F 23 460-479 5 5 5 9 9 9 F F F 24 480-499 4 4 4 8 8 8 F F F 25 500-519 4 4 4 4 4 4 F F F 26 520-539 3 3 3 3 3 3 F F F 27 540-559 3 3 3 3 3 3 F F F 28 560-579 4 4 4 6 6 6 F F F 29 580-599 5 5 5 9 9 9 F F F Max gen: 5, Max flush: 9 Begin loop 3 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:38.568574097Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1157,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:53:38.575691777Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1157,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:38.575759897Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1157,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:38.582760694Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1157,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:38.583072108Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1157} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:38.583099234Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1157} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:38.583107833Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1157} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:38.583119651Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1157} {"msg":"Crucible 8213e320-ea9a-4791-ba2f-344e1d32bda4 has session id: 15594769-edd8-4a1a-a0bd-e93c38b2056f","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:38.583405879Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1157,"session_id":"15594769-edd8-4a1a-a0bd-e93c38b2056f"} {"msg":"Upstairs opts: Upstairs UUID: 8213e320-ea9a-4791-ba2f-344e1d32bda4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:38.583437284Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1157,"session_id":"15594769-edd8-4a1a-a0bd-e93c38b2056f"} {"msg":"Crucible stats registered with UUID: 8213e320-ea9a-4791-ba2f-344e1d32bda4","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:38.583448472Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1157,"session_id":"15594769-edd8-4a1a-a0bd-e93c38b2056f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:53:38.583457771Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1157,"session_id":"15594769-edd8-4a1a-a0bd-e93c38b2056f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:38.58360032Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1157,"":"io task","client":"0","":"downstairs","session_id":"15594769-edd8-4a1a-a0bd-e93c38b2056f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:38.583617377Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1157,"":"io task","client":"2","":"downstairs","session_id":"15594769-edd8-4a1a-a0bd-e93c38b2056f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:38.583627556Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1157,"":"io task","client":"1","":"downstairs","session_id":"15594769-edd8-4a1a-a0bd-e93c38b2056f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:38.58380208Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1157,"task":"crutest"} {"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:38.583818798Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1157} {"msg":"8213e320-ea9a-4791-ba2f-344e1d32bda4 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:38.583827046Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1157,"session_id":"15594769-edd8-4a1a-a0bd-e93c38b2056f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:38.583835775Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1157,"client":"0","":"downstairs","session_id":"15594769-edd8-4a1a-a0bd-e93c38b2056f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:38.583845514Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1157,"client":"1","":"downstairs","session_id":"15594769-edd8-4a1a-a0bd-e93c38b2056f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:38.583855242Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1157,"client":"2","":"downstairs","session_id":"15594769-edd8-4a1a-a0bd-e93c38b2056f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:38.583864851Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1157,"":"io task","client":"2","":"downstairs","session_id":"15594769-edd8-4a1a-a0bd-e93c38b2056f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:38.583874939Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1157,"":"io task","client":"0","":"downstairs","session_id":"15594769-edd8-4a1a-a0bd-e93c38b2056f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:38.583885018Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1157,"":"io task","client":"1","":"downstairs","session_id":"15594769-edd8-4a1a-a0bd-e93c38b2056f"} {"msg":"ds_connection connected from Ok(127.0.0.1:61168)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:38.583928761Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1157,"":"io task","client":"2","":"downstairs","session_id":"15594769-edd8-4a1a-a0bd-e93c38b2056f"} {"msg":"ds_connection connected from Ok(127.0.0.1:47089)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:38.58393979Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1157,"":"io task","client":"0","":"downstairs","session_id":"15594769-edd8-4a1a-a0bd-e93c38b2056f"} {"msg":"ds_connection connected from Ok(127.0.0.1:56247)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:38.583950098Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1157,"":"io task","client":"1","":"downstairs","session_id":"15594769-edd8-4a1a-a0bd-e93c38b2056f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:38.584562428Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1157,"client":"1","":"downstairs","session_id":"15594769-edd8-4a1a-a0bd-e93c38b2056f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:38.584605132Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1157,"client":"2","":"downstairs","session_id":"15594769-edd8-4a1a-a0bd-e93c38b2056f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:38.584657174Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1157,"client":"0","":"downstairs","session_id":"15594769-edd8-4a1a-a0bd-e93c38b2056f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:38.584865293Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1157,"session_id":"15594769-edd8-4a1a-a0bd-e93c38b2056f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:38.584891609Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1157,"session_id":"15594769-edd8-4a1a-a0bd-e93c38b2056f"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 9, 5, 5, 1, 5, 5, 3, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:38.584960729Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1157,"":"downstairs","":"downstairs","session_id":"15594769-edd8-4a1a-a0bd-e93c38b2056f"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 5, 4, 4, 1, 4, 4, 3, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:38.584971508Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1157,"":"downstairs","":"downstairs","session_id":"15594769-edd8-4a1a-a0bd-e93c38b2056f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:38.584982496Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1157,"":"downstairs","":"downstairs","session_id":"15594769-edd8-4a1a-a0bd-e93c38b2056f"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 9, 5, 5, 1, 5, 5, 3, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:38.584992775Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1157,"":"downstairs","":"downstairs","session_id":"15594769-edd8-4a1a-a0bd-e93c38b2056f"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 5, 4, 4, 1, 4, 4, 3, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:38.585014661Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1157,"":"downstairs","":"downstairs","session_id":"15594769-edd8-4a1a-a0bd-e93c38b2056f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:38.58502559Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1157,"":"downstairs","":"downstairs","session_id":"15594769-edd8-4a1a-a0bd-e93c38b2056f"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 9, 5, 5, 1, 5, 5, 3, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:38.585036118Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1157,"":"downstairs","":"downstairs","session_id":"15594769-edd8-4a1a-a0bd-e93c38b2056f"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 5, 4, 4, 1, 4, 4, 3, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:38.585046377Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1157,"":"downstairs","":"downstairs","session_id":"15594769-edd8-4a1a-a0bd-e93c38b2056f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:38.585057285Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1157,"":"downstairs","":"downstairs","session_id":"15594769-edd8-4a1a-a0bd-e93c38b2056f"} {"msg":"Max found gen is 6","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:38.585066744Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1157,"":"downstairs","":"downstairs","session_id":"15594769-edd8-4a1a-a0bd-e93c38b2056f"} {"msg":"Generation requested: 6 >= found:6","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:38.585076262Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1157,"":"downstairs","":"downstairs","session_id":"15594769-edd8-4a1a-a0bd-e93c38b2056f"} {"msg":"Next flush: 10","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:38.585087361Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1157,"":"downstairs","":"downstairs","session_id":"15594769-edd8-4a1a-a0bd-e93c38b2056f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:38.585096619Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1157,"":"downstairs","":"downstairs","session_id":"15594769-edd8-4a1a-a0bd-e93c38b2056f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:38.585106368Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1157,"session_id":"15594769-edd8-4a1a-a0bd-e93c38b2056f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:38.585115066Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1157,"client":"0","":"downstairs","session_id":"15594769-edd8-4a1a-a0bd-e93c38b2056f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:38.585124545Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1157,"client":"1","":"downstairs","session_id":"15594769-edd8-4a1a-a0bd-e93c38b2056f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:38.585134034Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1157,"client":"2","":"downstairs","session_id":"15594769-edd8-4a1a-a0bd-e93c38b2056f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:38.585143902Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1157,"session_id":"15594769-edd8-4a1a-a0bd-e93c38b2056f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:38.585152981Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1157,"session_id":"15594769-edd8-4a1a-a0bd-e93c38b2056f"} {"msg":"8213e320-ea9a-4791-ba2f-344e1d32bda4 is now active with session: 15594769-edd8-4a1a-a0bd-e93c38b2056f","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:38.585162819Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1157,"session_id":"15594769-edd8-4a1a-a0bd-e93c38b2056f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:38.585171698Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1157,"session_id":"15594769-edd8-4a1a-a0bd-e93c38b2056f"} {"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:38.585180817Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1157} {"msg":"Activated sub_volume 8213e320-ea9a-4791-ba2f-344e1d32bda4","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:38.585188826Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1157} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:38.585197084Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1157,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:38.585506529Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1157,"task":"crutest"} 01/30 Write block 581 len 20480 data: 2 2 2 2 2 02/30 Read block 343 len 4096 03/30 Read block 239 len 12288 04/30 Write block 481 len 8192 data: 3 3 05/30 Write block 25 len 4096 data: 2 06/30 Read block 10 len 40960 07/30 Read block 304 len 24576 08/30 Write block 410 len 12288 data: 2 2 2 09/30 Read block 48 len 12288 10/30 Read block 266 len 8192 11/30 Write block 284 len 16384 data: 2 2 2 2 12/30 Write block 9 len 24576 data: 2 2 2 2 2 2 13/30 Read block 331 len 36864 14/30 Write block 410 len 32768 data: 3 3 3 2 2 2 2 2 15/30 Read block 377 len 20480 16/30 Write block 198 len 16384 data: 3 2 2 2 17/30 Read block 494 len 16384 18/30 Write block 34 len 32768 data: 2 2 2 2 2 2 2 2 19/30 Read block 235 len 36864 20/30 Read block 109 len 28672 21/30 Read block 186 len 12288 22/30 Write block 43 len 4096 data: 2 23/30 Write block 173 len 24576 data: 3 3 3 3 3 3 24/30 Read block 134 len 16384 25/30 Write block 258 len 24576 data: 2 2 2 2 2 3 26/30 Read block 88 len 28672 27/30 Write block 15 len 32768 data: 2 2 2 2 2 2 2 2 28/30 Read block 102 len 36864 29/30 Write block 330 len 4096 data: 2 30/30 Write block 236 len 12288 data: 2 2 2 ---------------------------------------------------------------- Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Sent Done Done false 1001 Acked Read 1 Sent Done Done false 1002 Acked Read 3 Sent Done Done false 1003 Acked Write 2 Sent Done Done false 1004 Acked Write 1 Sent Done Done false 1005 Acked Read 10 Sent Done Done false 1006 Acked Read 6 Sent Done Done false 1007 Acked Write 3 Sent Done Done false 1008 Acked Read 3 Sent Done Done false 1009 Acked Read 2 Sent Done Done false 1010 Acked Write 4 Sent Done Done false 1011 Acked Write 6 Sent Done Done false 1012 Acked Read 9 Sent Done Done false 1013 Acked Write 8 Sent Done Done false 1014 Acked Read 5 Sent Done Done false 1015 Acked Write 4 Sent Done Done false 1016 Acked Read 4 Sent Done Done false 1017 Acked Write 8 Sent Done Done false 1018 Acked Read 9 Sent Done Done false 1019 Acked Read 7 Sent Done Done false 1020 Acked Read 3 Sent Done Done false 1021 Acked Write 1 Sent Done Done false 1022 Acked Write 6 Sent Done Done false 1023 Acked Read 4 Sent Done Done false 1024 Acked Write 6 Sent Done Done false 1025 Acked Read 7 Sent Done Done false 1026 Acked Write 8 Sent Done Done false 1027 Acked Read 9 Sent Done Done false 1028 Acked Write 1 Sent Sent Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 2 1 33 Done 0 28 29 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:53:38.593776449Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1157} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:38.593951143Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1157,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F T T <--- 1 020-039 1 1 1 1 1 1 F T T <--- 2 040-059 1 1 1 1 1 1 F T T <--- 3 060-079 1 1 1 1 1 1 F F F 4 080-099 5 5 5 9 9 9 F F F 5 100-119 4 4 4 5 5 5 F F F 6 120-139 4 4 4 5 5 5 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 4 4 4 5 5 5 F T T <--- 9 180-199 4 4 4 5 5 5 F T T <--- 10 200-219 3 3 3 3 3 3 F T T <--- 11 220-239 1 1 1 1 1 1 F T T <--- 12 240-259 1 1 1 1 1 1 F T T <--- 13 260-279 2 2 2 2 2 2 F T T <--- 14 280-299 1 1 1 1 1 1 F T T <--- 15 300-319 3 3 3 3 3 3 F F F 16 320-339 4 4 4 5 5 5 F T T <--- 17 340-359 5 5 5 9 9 9 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 3 3 3 3 3 3 F F F 20 400-419 1 1 1 1 1 1 F T T <--- 21 420-439 1 1 1 1 1 1 F F F 22 440-459 2 2 2 2 2 2 F F F 23 460-479 5 5 5 9 9 9 F F F 24 480-499 4 4 4 8 8 8 F T T <--- 25 500-519 4 4 4 4 4 4 F F F 26 520-539 3 3 3 3 3 3 F F F 27 540-559 3 3 3 3 3 3 F F F 28 560-579 4 4 4 6 6 6 F F F 29 580-599 5 5 5 9 9 9 F T T <--- Max gen: 5, Max flush: 9 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 3 Verifying data now No change after verify Loop: 3 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 6 6 6 10 10 10 F F F 1 020-039 6 6 6 10 10 10 F F F 2 040-059 6 6 6 10 10 10 F F F 3 060-079 1 1 1 1 1 1 F F F 4 080-099 5 5 5 9 9 9 F F F 5 100-119 4 4 4 5 5 5 F F F 6 120-139 4 4 4 5 5 5 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 6 6 6 10 10 10 F F F 9 180-199 6 6 6 10 10 10 F F F 10 200-219 6 6 6 10 10 10 F F F 11 220-239 6 6 6 10 10 10 F F F 12 240-259 6 6 6 10 10 10 F F F 13 260-279 6 6 6 10 10 10 F F F 14 280-299 6 6 6 10 10 10 F F F 15 300-319 3 3 3 3 3 3 F F F 16 320-339 6 6 6 10 10 10 F F F 17 340-359 5 5 5 9 9 9 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 3 3 3 3 3 3 F F F 20 400-419 6 6 6 10 10 10 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 2 2 2 2 2 2 F F F 23 460-479 5 5 5 9 9 9 F F F 24 480-499 6 6 6 10 10 10 F F F 25 500-519 4 4 4 4 4 4 F F F 26 520-539 3 3 3 3 3 3 F F F 27 540-559 3 3 3 3 3 3 F F F 28 560-579 4 4 4 6 6 6 F F F 29 580-599 6 6 6 10 10 10 F F F Max gen: 6, Max flush: 10 Begin loop 4 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:42.848575843Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1165,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:53:42.855671117Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1165,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:42.855722239Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1165,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:42.862416971Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1165,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:42.862669634Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1165} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:42.86269807Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1165} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:42.862706919Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1165} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:42.862718617Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1165} {"msg":"Crucible b6f7f86a-a040-4568-8559-5d96ba038ed8 has session id: c27f549b-32aa-4e58-b20b-fa995ce5aad0","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:42.863048358Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1165,"session_id":"c27f549b-32aa-4e58-b20b-fa995ce5aad0"} {"msg":"Upstairs opts: Upstairs UUID: b6f7f86a-a040-4568-8559-5d96ba038ed8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:42.863079264Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1165,"session_id":"c27f549b-32aa-4e58-b20b-fa995ce5aad0"} {"msg":"Crucible stats registered with UUID: b6f7f86a-a040-4568-8559-5d96ba038ed8","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:42.863089832Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1165,"session_id":"c27f549b-32aa-4e58-b20b-fa995ce5aad0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:53:42.863098961Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1165,"session_id":"c27f549b-32aa-4e58-b20b-fa995ce5aad0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:42.863217133Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1165,"":"io task","client":"0","":"downstairs","session_id":"c27f549b-32aa-4e58-b20b-fa995ce5aad0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:42.86324039Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1165,"":"io task","client":"1","":"downstairs","session_id":"c27f549b-32aa-4e58-b20b-fa995ce5aad0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:42.863251138Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1165,"":"io task","client":"2","":"downstairs","session_id":"c27f549b-32aa-4e58-b20b-fa995ce5aad0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:42.863406465Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1165,"task":"crutest"} {"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:42.863424813Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1165} {"msg":"b6f7f86a-a040-4568-8559-5d96ba038ed8 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:42.86344189Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1165,"session_id":"c27f549b-32aa-4e58-b20b-fa995ce5aad0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:42.863452509Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1165,"client":"0","":"downstairs","session_id":"c27f549b-32aa-4e58-b20b-fa995ce5aad0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:42.863462397Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1165,"client":"1","":"downstairs","session_id":"c27f549b-32aa-4e58-b20b-fa995ce5aad0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:42.863472256Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1165,"client":"2","":"downstairs","session_id":"c27f549b-32aa-4e58-b20b-fa995ce5aad0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:42.863482034Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1165,"":"io task","client":"2","":"downstairs","session_id":"c27f549b-32aa-4e58-b20b-fa995ce5aad0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:42.863492353Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1165,"":"io task","client":"0","":"downstairs","session_id":"c27f549b-32aa-4e58-b20b-fa995ce5aad0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:42.863502441Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1165,"":"io task","client":"1","":"downstairs","session_id":"c27f549b-32aa-4e58-b20b-fa995ce5aad0"} {"msg":"ds_connection connected from Ok(127.0.0.1:50629)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:42.863536006Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1165,"":"io task","client":"2","":"downstairs","session_id":"c27f549b-32aa-4e58-b20b-fa995ce5aad0"} {"msg":"ds_connection connected from Ok(127.0.0.1:43840)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:42.863553664Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1165,"":"io task","client":"0","":"downstairs","session_id":"c27f549b-32aa-4e58-b20b-fa995ce5aad0"} {"msg":"ds_connection connected from Ok(127.0.0.1:43273)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:42.863690923Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1165,"":"io task","client":"1","":"downstairs","session_id":"c27f549b-32aa-4e58-b20b-fa995ce5aad0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:42.864234313Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1165,"client":"1","":"downstairs","session_id":"c27f549b-32aa-4e58-b20b-fa995ce5aad0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:42.86425471Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1165,"client":"2","":"downstairs","session_id":"c27f549b-32aa-4e58-b20b-fa995ce5aad0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:42.864266129Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1165,"client":"0","":"downstairs","session_id":"c27f549b-32aa-4e58-b20b-fa995ce5aad0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:42.864549937Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1165,"session_id":"c27f549b-32aa-4e58-b20b-fa995ce5aad0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:42.864582622Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1165,"session_id":"c27f549b-32aa-4e58-b20b-fa995ce5aad0"} {"msg":"[0]R flush_numbers[0..12]: [10, 10, 10, 1, 9, 5, 5, 1, 10, 10, 10, 10]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:42.864647812Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1165,"":"downstairs","":"downstairs","session_id":"c27f549b-32aa-4e58-b20b-fa995ce5aad0"} {"msg":"[0]R generation[0..12]: [6, 6, 6, 1, 5, 4, 4, 1, 6, 6, 6, 6]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:42.86466082Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1165,"":"downstairs","":"downstairs","session_id":"c27f549b-32aa-4e58-b20b-fa995ce5aad0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:42.864671869Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1165,"":"downstairs","":"downstairs","session_id":"c27f549b-32aa-4e58-b20b-fa995ce5aad0"} {"msg":"[1]R flush_numbers[0..12]: [10, 10, 10, 1, 9, 5, 5, 1, 10, 10, 10, 10]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:42.864682237Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1165,"":"downstairs","":"downstairs","session_id":"c27f549b-32aa-4e58-b20b-fa995ce5aad0"} {"msg":"[1]R generation[0..12]: [6, 6, 6, 1, 5, 4, 4, 1, 6, 6, 6, 6]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:42.864692546Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1165,"":"downstairs","":"downstairs","session_id":"c27f549b-32aa-4e58-b20b-fa995ce5aad0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:42.864703404Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1165,"":"downstairs","":"downstairs","session_id":"c27f549b-32aa-4e58-b20b-fa995ce5aad0"} {"msg":"[2]R flush_numbers[0..12]: [10, 10, 10, 1, 9, 5, 5, 1, 10, 10, 10, 10]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:42.864713742Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1165,"":"downstairs","":"downstairs","session_id":"c27f549b-32aa-4e58-b20b-fa995ce5aad0"} {"msg":"[2]R generation[0..12]: [6, 6, 6, 1, 5, 4, 4, 1, 6, 6, 6, 6]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:42.864723881Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1165,"":"downstairs","":"downstairs","session_id":"c27f549b-32aa-4e58-b20b-fa995ce5aad0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:42.864734659Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1165,"":"downstairs","":"downstairs","session_id":"c27f549b-32aa-4e58-b20b-fa995ce5aad0"} {"msg":"Max found gen is 7","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:42.864744198Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1165,"":"downstairs","":"downstairs","session_id":"c27f549b-32aa-4e58-b20b-fa995ce5aad0"} {"msg":"Generation requested: 8 >= found:7","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:42.864753817Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1165,"":"downstairs","":"downstairs","session_id":"c27f549b-32aa-4e58-b20b-fa995ce5aad0"} {"msg":"Next flush: 11","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:42.864763185Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1165,"":"downstairs","":"downstairs","session_id":"c27f549b-32aa-4e58-b20b-fa995ce5aad0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:42.864772734Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1165,"":"downstairs","":"downstairs","session_id":"c27f549b-32aa-4e58-b20b-fa995ce5aad0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:42.864782582Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1165,"session_id":"c27f549b-32aa-4e58-b20b-fa995ce5aad0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:42.864791341Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1165,"client":"0","":"downstairs","session_id":"c27f549b-32aa-4e58-b20b-fa995ce5aad0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:42.86480093Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1165,"client":"1","":"downstairs","session_id":"c27f549b-32aa-4e58-b20b-fa995ce5aad0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:42.864812438Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1165,"client":"2","":"downstairs","session_id":"c27f549b-32aa-4e58-b20b-fa995ce5aad0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:42.864822346Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1165,"session_id":"c27f549b-32aa-4e58-b20b-fa995ce5aad0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:42.864831625Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1165,"session_id":"c27f549b-32aa-4e58-b20b-fa995ce5aad0"} {"msg":"b6f7f86a-a040-4568-8559-5d96ba038ed8 is now active with session: c27f549b-32aa-4e58-b20b-fa995ce5aad0","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:42.864841684Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1165,"session_id":"c27f549b-32aa-4e58-b20b-fa995ce5aad0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:42.864850532Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1165,"session_id":"c27f549b-32aa-4e58-b20b-fa995ce5aad0"} {"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:42.864859581Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1165} {"msg":"Activated sub_volume b6f7f86a-a040-4568-8559-5d96ba038ed8","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:42.86486752Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1165} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:42.864875739Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1165,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:42.865222567Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1165,"task":"crutest"} 01/30 Write block 359 len 28672 data: 2 2 2 2 2 2 2 02/30 Read block 507 len 24576 03/30 Read block 230 len 4096 04/30 Read block 347 len 24576 05/30 Flush 06/30 Flush 07/30 Write block 107 len 8192 data: 2 2 08/30 Read block 488 len 40960 09/30 Read block 65 len 32768 10/30 Read block 563 len 36864 11/30 Read block 306 len 36864 12/30 Read block 154 len 8192 13/30 Read block 497 len 20480 14/30 Write block 22 len 32768 data: 3 2 2 3 2 2 2 2 15/30 Write block 85 len 12288 data: 2 2 2 16/30 Flush 17/30 Write block 584 len 12288 data: 3 3 2 18/30 Read block 241 len 36864 19/30 Flush 20/30 Flush 21/30 Flush 22/30 Write block 88 len 12288 data: 3 3 3 23/30 Read block 354 len 12288 24/30 Write block 224 len 4096 data: 2 25/30 Read block 64 len 20480 26/30 Write block 84 len 32768 data: 2 3 3 3 4 4 4 3 27/30 Write block 237 len 40960 data: 3 3 2 2 2 2 2 2 2 2 28/30 Write block 513 len 4096 data: 3 29/30 Write block 593 len 12288 data: 3 3 2 30/30 Write block 448 len 20480 data: 2 2 3 3 3 ---------------------------------------------------------------- Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Read 6 Sent Done Done false 1002 Acked Read 1 Sent Done Done false 1003 Acked Read 6 Sent Done Done false 1004 Acked Flush 0 Sent Done Done false 1005 Acked Flush 0 Sent Done Done false 1006 Acked Write 2 Sent Done Done false 1007 Acked Read 10 Sent Done Done false 1008 Acked Read 8 Sent Done Done false 1009 Acked Read 9 Sent Done Done false 1010 Acked Read 9 Sent Done Done false 1011 Acked Read 2 Sent Done Done false 1012 Acked Read 5 Sent Done Done false 1013 Acked Write 8 Sent Done Done false 1014 Acked Write 3 Sent Done Done false 1015 Acked Flush 0 Sent Done Done false 1016 Acked Write 3 Sent Done Done false 1017 Acked Read 9 Sent Done Done false 1018 Acked Flush 0 Sent Done Done false 1019 Acked Flush 0 Sent Done Done false 1020 Acked Flush 0 Sent Done Done false 1021 Acked Write 3 Sent Done Done false 1022 Acked Read 3 Sent Done Done false 1023 Acked Write 1 Sent Done Done false 1024 Acked Read 5 Sent Done Done false 1025 Acked Write 8 Sent Done Done false 1026 Acked Write 10 Sent Sent Sent false 1027 Acked Write 1 Sent Sent Sent false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 4 4 37 Done 1 26 26 53 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:53:42.872982822Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1165} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:42.873126571Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1165,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 6 6 6 10 10 10 F F F 1 020-039 6 8 8 10 13 13 F F F <--- 2 040-059 6 6 6 10 10 10 F F F 3 060-079 1 1 1 1 1 1 F F F 4 080-099 5 8 8 9 13 13 F T T <--- 5 100-119 4 8 8 5 13 13 F F F <--- 6 120-139 4 4 4 5 5 5 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 6 6 6 10 10 10 F F F 9 180-199 6 6 6 10 10 10 F F F 10 200-219 6 6 6 10 10 10 F F F 11 220-239 6 6 6 10 10 10 F T T <--- 12 240-259 6 6 6 10 10 10 F T T <--- 13 260-279 6 6 6 10 10 10 F F F 14 280-299 6 6 6 10 10 10 F F F 15 300-319 3 3 3 3 3 3 F F F 16 320-339 6 6 6 10 10 10 F F F 17 340-359 5 8 8 9 11 11 T F F <--- 18 360-379 1 8 8 1 11 11 T F F <--- 19 380-399 3 3 3 3 3 3 F F F 20 400-419 6 6 6 10 10 10 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 2 2 2 2 2 2 F T T <--- 23 460-479 5 5 5 9 9 9 F F F 24 480-499 6 6 6 10 10 10 F F F 25 500-519 4 4 4 4 4 4 F T T <--- 26 520-539 3 3 3 3 3 3 F F F 27 540-559 3 3 3 3 3 3 F F F 28 560-579 4 4 4 6 6 6 F F F 29 580-599 6 8 8 10 14 14 F T T <--- Max gen: 8, Max flush: 14 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 4 Verifying data now No change after verify Loop: 4 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 6 6 6 10 10 10 F F F 1 020-039 8 8 8 13 13 13 F F F 2 040-059 6 6 6 10 10 10 F F F 3 060-079 1 1 1 1 1 1 F F F 4 080-099 9 9 9 15 15 15 F F F 5 100-119 8 8 8 13 13 13 F F F 6 120-139 4 4 4 5 5 5 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 6 6 6 10 10 10 F F F 9 180-199 6 6 6 10 10 10 F F F 10 200-219 6 6 6 10 10 10 F F F 11 220-239 9 9 9 15 15 15 F F F 12 240-259 9 9 9 15 15 15 F F F 13 260-279 6 6 6 10 10 10 F F F 14 280-299 6 6 6 10 10 10 F F F 15 300-319 3 3 3 3 3 3 F F F 16 320-339 6 6 6 10 10 10 F F F 17 340-359 8 8 8 11 11 11 F F F 18 360-379 8 8 8 11 11 11 F F F 19 380-399 3 3 3 3 3 3 F F F 20 400-419 6 6 6 10 10 10 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 9 9 9 15 15 15 F F F 23 460-479 5 5 5 9 9 9 F F F 24 480-499 6 6 6 10 10 10 F F F 25 500-519 9 9 9 15 15 15 F F F 26 520-539 3 3 3 3 3 3 F F F 27 540-559 3 3 3 3 3 3 F F F 28 560-579 4 4 4 6 6 6 F F F 29 580-599 9 9 9 15 15 15 F F F Max gen: 9, Max flush: 15 Begin loop 5 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:47.119078282Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1173,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:47.127323696Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1173,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:47.127618142Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1173} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:47.127648688Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1173} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:47.127657346Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1173} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:47.127668725Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1173} {"msg":"Crucible 19a45364-957d-41bb-862b-294ed446d546 has session id: 66b6aba7-107f-45fe-9820-0832b7874e05","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:47.128017743Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1173,"session_id":"66b6aba7-107f-45fe-9820-0832b7874e05"} {"msg":"Upstairs opts: Upstairs UUID: 19a45364-957d-41bb-862b-294ed446d546, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:47.128049199Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1173,"session_id":"66b6aba7-107f-45fe-9820-0832b7874e05"} {"msg":"Crucible stats registered with UUID: 19a45364-957d-41bb-862b-294ed446d546","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:47.128060467Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1173,"session_id":"66b6aba7-107f-45fe-9820-0832b7874e05"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:53:47.128069616Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1173,"session_id":"66b6aba7-107f-45fe-9820-0832b7874e05"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:47.128203796Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1173,"":"io task","client":"1","":"downstairs","session_id":"66b6aba7-107f-45fe-9820-0832b7874e05"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:47.128221723Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1173,"":"io task","client":"0","":"downstairs","session_id":"66b6aba7-107f-45fe-9820-0832b7874e05"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:47.128231592Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1173,"":"io task","client":"2","":"downstairs","session_id":"66b6aba7-107f-45fe-9820-0832b7874e05"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:47.128356163Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1173,"task":"crutest"} {"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:47.128374971Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1173} {"msg":"19a45364-957d-41bb-862b-294ed446d546 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:47.128407576Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1173,"session_id":"66b6aba7-107f-45fe-9820-0832b7874e05"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:47.128417024Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1173,"client":"0","":"downstairs","session_id":"66b6aba7-107f-45fe-9820-0832b7874e05"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:47.128426913Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1173,"client":"1","":"downstairs","session_id":"66b6aba7-107f-45fe-9820-0832b7874e05"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:47.128436472Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1173,"client":"2","":"downstairs","session_id":"66b6aba7-107f-45fe-9820-0832b7874e05"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:47.12844612Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1173,"":"io task","client":"2","":"downstairs","session_id":"66b6aba7-107f-45fe-9820-0832b7874e05"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:47.128456489Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1173,"":"io task","client":"1","":"downstairs","session_id":"66b6aba7-107f-45fe-9820-0832b7874e05"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:47.128490044Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1173,"":"io task","client":"0","":"downstairs","session_id":"66b6aba7-107f-45fe-9820-0832b7874e05"} {"msg":"ds_connection connected from Ok(127.0.0.1:48054)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:47.128504641Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1173,"":"io task","client":"1","":"downstairs","session_id":"66b6aba7-107f-45fe-9820-0832b7874e05"} {"msg":"ds_connection connected from Ok(127.0.0.1:40157)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:47.12851646Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1173,"":"io task","client":"2","":"downstairs","session_id":"66b6aba7-107f-45fe-9820-0832b7874e05"} {"msg":"ds_connection connected from Ok(127.0.0.1:61222)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:47.128562053Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1173,"":"io task","client":"0","":"downstairs","session_id":"66b6aba7-107f-45fe-9820-0832b7874e05"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:47.129232114Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1173,"client":"1","":"downstairs","session_id":"66b6aba7-107f-45fe-9820-0832b7874e05"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:47.12925884Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1173,"client":"2","":"downstairs","session_id":"66b6aba7-107f-45fe-9820-0832b7874e05"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:47.129270858Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1173,"client":"0","":"downstairs","session_id":"66b6aba7-107f-45fe-9820-0832b7874e05"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:47.129568235Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1173,"session_id":"66b6aba7-107f-45fe-9820-0832b7874e05"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:47.12960163Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1173,"session_id":"66b6aba7-107f-45fe-9820-0832b7874e05"} {"msg":"[0]R flush_numbers[0..12]: [10, 13, 10, 1, 15, 13, 5, 1, 10, 10, 10, 15]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:47.129650662Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1173,"":"downstairs","":"downstairs","session_id":"66b6aba7-107f-45fe-9820-0832b7874e05"} {"msg":"[0]R generation[0..12]: [6, 8, 6, 1, 9, 8, 4, 1, 6, 6, 6, 9]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:47.129662101Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1173,"":"downstairs","":"downstairs","session_id":"66b6aba7-107f-45fe-9820-0832b7874e05"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:47.129673409Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1173,"":"downstairs","":"downstairs","session_id":"66b6aba7-107f-45fe-9820-0832b7874e05"} {"msg":"[1]R flush_numbers[0..12]: [10, 13, 10, 1, 15, 13, 5, 1, 10, 10, 10, 15]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:47.129683987Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1173,"":"downstairs","":"downstairs","session_id":"66b6aba7-107f-45fe-9820-0832b7874e05"} {"msg":"[1]R generation[0..12]: [6, 8, 6, 1, 9, 8, 4, 1, 6, 6, 6, 9]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:47.129694296Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1173,"":"downstairs","":"downstairs","session_id":"66b6aba7-107f-45fe-9820-0832b7874e05"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:47.129705444Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1173,"":"downstairs","":"downstairs","session_id":"66b6aba7-107f-45fe-9820-0832b7874e05"} {"msg":"[2]R flush_numbers[0..12]: [10, 13, 10, 1, 15, 13, 5, 1, 10, 10, 10, 15]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:47.129715803Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1173,"":"downstairs","":"downstairs","session_id":"66b6aba7-107f-45fe-9820-0832b7874e05"} {"msg":"[2]R generation[0..12]: [6, 8, 6, 1, 9, 8, 4, 1, 6, 6, 6, 9]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:47.129728391Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1173,"":"downstairs","":"downstairs","session_id":"66b6aba7-107f-45fe-9820-0832b7874e05"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:47.129739549Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1173,"":"downstairs","":"downstairs","session_id":"66b6aba7-107f-45fe-9820-0832b7874e05"} {"msg":"Max found gen is 10","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:47.129749208Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1173,"":"downstairs","":"downstairs","session_id":"66b6aba7-107f-45fe-9820-0832b7874e05"} {"msg":"Generation requested: 10 >= found:10","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:47.129758936Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1173,"":"downstairs","":"downstairs","session_id":"66b6aba7-107f-45fe-9820-0832b7874e05"} {"msg":"Next flush: 16","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:47.129768345Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1173,"":"downstairs","":"downstairs","session_id":"66b6aba7-107f-45fe-9820-0832b7874e05"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:47.129777884Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1173,"":"downstairs","":"downstairs","session_id":"66b6aba7-107f-45fe-9820-0832b7874e05"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:47.129787932Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1173,"session_id":"66b6aba7-107f-45fe-9820-0832b7874e05"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:47.129796821Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1173,"client":"0","":"downstairs","session_id":"66b6aba7-107f-45fe-9820-0832b7874e05"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:47.129806819Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1173,"client":"1","":"downstairs","session_id":"66b6aba7-107f-45fe-9820-0832b7874e05"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:47.129816788Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1173,"client":"2","":"downstairs","session_id":"66b6aba7-107f-45fe-9820-0832b7874e05"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:47.129826806Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1173,"session_id":"66b6aba7-107f-45fe-9820-0832b7874e05"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:47.129836085Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1173,"session_id":"66b6aba7-107f-45fe-9820-0832b7874e05"} {"msg":"19a45364-957d-41bb-862b-294ed446d546 is now active with session: 66b6aba7-107f-45fe-9820-0832b7874e05","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:47.129845944Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1173,"session_id":"66b6aba7-107f-45fe-9820-0832b7874e05"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:47.129854602Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1173,"session_id":"66b6aba7-107f-45fe-9820-0832b7874e05"} {"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:47.129863891Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1173} {"msg":"Activated sub_volume 19a45364-957d-41bb-862b-294ed446d546","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:47.12987191Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1173} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:47.129880029Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1173,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:47.130197642Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1173,"task":"crutest"} 01/30 Write block 207 len 40960 data: 2 2 3 2 2 2 2 2 2 2 02/30 Write block 438 len 4096 data: 2 03/30 Read block 477 len 28672 04/30 Read block 173 len 28672 05/30 Write block 35 len 40960 data: 3 3 3 3 3 3 3 2 3 2 06/30 Read block 20 len 36864 07/30 Read block 308 len 20480 08/30 Write block 198 len 40960 data: 4 3 3 3 2 2 2 2 2 3 09/30 Write block 275 len 16384 data: 2 2 2 2 10/30 Write block 444 len 12288 data: 2 2 2 11/30 Read block 394 len 28672 12/30 Read block 591 len 24576 13/30 Write block 17 len 32768 data: 3 3 3 3 3 4 3 3 14/30 Read block 519 len 8192 15/30 Read block 32 len 4096 16/30 Flush 17/30 Flush 18/30 Write block 545 len 4096 data: 2 19/30 Flush 20/30 Flush 21/30 Write block 47 len 40960 data: 2 2 2 2 2 2 2 2 2 2 22/30 Flush 23/30 Write block 69 len 40960 data: 2 2 2 2 2 2 2 2 2 2 24/30 Write block 96 len 24576 data: 2 2 3 3 3 3 25/30 Read block 567 len 28672 26/30 Write block 239 len 28672 data: 3 3 3 3 3 3 3 27/30 Read block 252 len 40960 28/30 Read block 486 len 4096 29/30 Read block 190 len 28672 30/30 Write block 172 len 20480 data: 2 4 4 4 4 ---------------------------------------------------------------- Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Write 1 Done Done Done false 1002 Acked Read 7 Done Done Done false 1003 Acked Read 7 Done Done Done false 1004 Acked Write 10 Done Done Sent false 1005 Acked Read 9 Done Done Sent false 1006 Acked Read 5 Done Done Sent false 1007 Acked Write 10 Done Done Sent false 1008 Acked Write 4 Done Done Sent false 1009 Acked Write 3 Done Done Sent false 1010 Acked Read 7 Done Done Sent false 1011 Acked Read 6 Done Done Sent false 1012 Acked Write 8 Done Done Sent false 1013 Acked Read 2 Done Done Sent false 1014 Acked Read 1 Done Done Sent false 1015 Acked Flush 0 Done Done Sent false 1016 Acked Flush 0 Done Done Sent false 1017 Acked Write 1 Done Done Sent false 1018 Acked Flush 0 Done Done Sent false 1019 Acked Flush 0 Done Done Sent false 1020 Acked Write 10 Done Done Sent false 1021 Acked Flush 0 Done Done Sent false 1022 Acked Write 10 Done Done Sent false 1023 Acked Write 6 Done Done Sent false 1024 Acked Read 7 Done Done Sent false 1025 Acked Write 7 Done Done Sent false 1026 Acked Read 10 Done Done Sent false 1027 Acked Read 1 Done Done Sent false 1028 Acked Read 7 Done Done Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 26 28 Done 29 29 4 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:53:47.139142222Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1173} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:47.139304978Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1173,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 6 16 16 10 F F F <--- 1 020-039 10 10 8 16 16 13 F F F <--- 2 040-059 10 10 6 20 20 10 F F F <--- 3 060-079 1 1 1 1 1 1 T T F <--- 4 080-099 9 9 9 15 15 15 T T F <--- 5 100-119 8 8 8 13 13 13 T T F <--- 6 120-139 4 4 4 5 5 5 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 6 6 6 10 10 10 T T F <--- 9 180-199 10 10 6 16 16 10 F F F <--- 10 200-219 10 10 6 16 16 10 F F T <--- 11 220-239 9 9 9 15 15 15 T T F <--- 12 240-259 9 9 9 15 15 15 T T F <--- 13 260-279 10 10 6 16 16 10 F F F <--- 14 280-299 6 6 6 10 10 10 F F F 15 300-319 3 3 3 3 3 3 F F F 16 320-339 6 6 6 10 10 10 F F F 17 340-359 8 8 8 11 11 11 F F F 18 360-379 8 8 8 11 11 11 F F F 19 380-399 3 3 3 3 3 3 F F F 20 400-419 6 6 6 10 10 10 F F F 21 420-439 10 10 1 16 16 1 F F T <--- 22 440-459 10 10 9 16 16 15 F F F <--- 23 460-479 5 5 5 9 9 9 F F F 24 480-499 6 6 6 10 10 10 F F F 25 500-519 9 9 9 15 15 15 F F F 26 520-539 3 3 3 3 3 3 F F F 27 540-559 10 10 3 18 18 3 F F F <--- 28 560-579 4 4 4 6 6 6 F F F 29 580-599 9 9 9 15 15 15 F F F Max gen: 10, Max flush: 20 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 5 Verifying data now No change after verify Loop: 5 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 16 16 16 F F F 1 020-039 10 10 10 16 16 16 F F F 2 040-059 10 10 10 20 20 20 F F F 3 060-079 11 11 11 21 21 21 F F F 4 080-099 11 11 11 21 21 21 F F F 5 100-119 11 11 11 21 21 21 F F F 6 120-139 4 4 4 5 5 5 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 11 11 11 21 21 21 F F F 9 180-199 10 10 10 16 16 16 F F F 10 200-219 10 10 10 16 16 16 F F F 11 220-239 11 11 11 21 21 21 F F F 12 240-259 11 11 11 21 21 21 F F F 13 260-279 10 10 10 16 16 16 F F F 14 280-299 6 6 6 10 10 10 F F F 15 300-319 3 3 3 3 3 3 F F F 16 320-339 6 6 6 10 10 10 F F F 17 340-359 8 8 8 11 11 11 F F F 18 360-379 8 8 8 11 11 11 F F F 19 380-399 3 3 3 3 3 3 F F F 20 400-419 6 6 6 10 10 10 F F F 21 420-439 10 10 10 16 16 16 F F F 22 440-459 10 10 10 16 16 16 F F F 23 460-479 5 5 5 9 9 9 F F F 24 480-499 6 6 6 10 10 10 F F F 25 500-519 9 9 9 15 15 15 F F F 26 520-539 3 3 3 3 3 3 F F F 27 540-559 10 10 10 18 18 18 F F F 28 560-579 4 4 4 6 6 6 F F F 29 580-599 9 9 9 15 15 15 F F F Max gen: 11, Max flush: 21 Begin loop 6 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:51.387861585Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1181,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:51.395896879Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1181,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:51.396187576Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1181} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:51.396222261Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1181} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:51.39623149Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1181} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:51.396243188Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1181} {"msg":"Crucible a5d5a495-3110-42b1-92ef-e1f6780ca2d0 has session id: 80107972-d21e-4892-97ef-c0f45a4d8158","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:51.396562371Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1181,"session_id":"80107972-d21e-4892-97ef-c0f45a4d8158"} {"msg":"Upstairs opts: Upstairs UUID: a5d5a495-3110-42b1-92ef-e1f6780ca2d0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:51.396586807Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1181,"session_id":"80107972-d21e-4892-97ef-c0f45a4d8158"} {"msg":"Crucible stats registered with UUID: a5d5a495-3110-42b1-92ef-e1f6780ca2d0","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:51.396596796Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1181,"session_id":"80107972-d21e-4892-97ef-c0f45a4d8158"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:53:51.396605875Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1181,"session_id":"80107972-d21e-4892-97ef-c0f45a4d8158"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:51.396749533Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1181,"":"io task","client":"2","":"downstairs","session_id":"80107972-d21e-4892-97ef-c0f45a4d8158"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:51.396781429Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1181,"":"io task","client":"0","":"downstairs","session_id":"80107972-d21e-4892-97ef-c0f45a4d8158"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:51.396792547Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1181,"":"io task","client":"1","":"downstairs","session_id":"80107972-d21e-4892-97ef-c0f45a4d8158"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:51.39691009Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1181,"task":"crutest"} {"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:51.396927917Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1181} {"msg":"a5d5a495-3110-42b1-92ef-e1f6780ca2d0 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:51.396958942Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1181,"session_id":"80107972-d21e-4892-97ef-c0f45a4d8158"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:51.396968411Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1181,"client":"0","":"downstairs","session_id":"80107972-d21e-4892-97ef-c0f45a4d8158"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:51.39697809Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1181,"client":"1","":"downstairs","session_id":"80107972-d21e-4892-97ef-c0f45a4d8158"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:51.396987598Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1181,"client":"2","":"downstairs","session_id":"80107972-d21e-4892-97ef-c0f45a4d8158"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:51.397006615Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1181,"":"io task","client":"2","":"downstairs","session_id":"80107972-d21e-4892-97ef-c0f45a4d8158"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:51.397017024Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1181,"":"io task","client":"0","":"downstairs","session_id":"80107972-d21e-4892-97ef-c0f45a4d8158"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:51.397026882Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1181,"":"io task","client":"1","":"downstairs","session_id":"80107972-d21e-4892-97ef-c0f45a4d8158"} {"msg":"ds_connection connected from Ok(127.0.0.1:41840)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:51.397080415Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1181,"":"io task","client":"2","":"downstairs","session_id":"80107972-d21e-4892-97ef-c0f45a4d8158"} {"msg":"ds_connection connected from Ok(127.0.0.1:52803)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:51.39711131Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1181,"":"io task","client":"1","":"downstairs","session_id":"80107972-d21e-4892-97ef-c0f45a4d8158"} {"msg":"ds_connection connected from Ok(127.0.0.1:59569)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:51.397122958Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1181,"":"io task","client":"0","":"downstairs","session_id":"80107972-d21e-4892-97ef-c0f45a4d8158"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:51.397761064Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1181,"client":"2","":"downstairs","session_id":"80107972-d21e-4892-97ef-c0f45a4d8158"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:51.397806737Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1181,"client":"1","":"downstairs","session_id":"80107972-d21e-4892-97ef-c0f45a4d8158"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:51.39785962Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1181,"client":"0","":"downstairs","session_id":"80107972-d21e-4892-97ef-c0f45a4d8158"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:51.398096135Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1181,"session_id":"80107972-d21e-4892-97ef-c0f45a4d8158"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:51.398146047Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1181,"session_id":"80107972-d21e-4892-97ef-c0f45a4d8158"} {"msg":"[0]R flush_numbers[0..12]: [16, 16, 20, 21, 21, 21, 5, 1, 21, 16, 16, 21]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:51.398171294Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1181,"":"downstairs","":"downstairs","session_id":"80107972-d21e-4892-97ef-c0f45a4d8158"} {"msg":"[0]R generation[0..12]: [10, 10, 10, 11, 11, 11, 4, 1, 11, 10, 10, 11]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:51.398182322Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1181,"":"downstairs","":"downstairs","session_id":"80107972-d21e-4892-97ef-c0f45a4d8158"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:51.39819321Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1181,"":"downstairs","":"downstairs","session_id":"80107972-d21e-4892-97ef-c0f45a4d8158"} {"msg":"[1]R flush_numbers[0..12]: [16, 16, 20, 21, 21, 21, 5, 1, 21, 16, 16, 21]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:51.398203629Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1181,"":"downstairs","":"downstairs","session_id":"80107972-d21e-4892-97ef-c0f45a4d8158"} {"msg":"[1]R generation[0..12]: [10, 10, 10, 11, 11, 11, 4, 1, 11, 10, 10, 11]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:51.398213937Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1181,"":"downstairs","":"downstairs","session_id":"80107972-d21e-4892-97ef-c0f45a4d8158"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:51.398224906Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1181,"":"downstairs","":"downstairs","session_id":"80107972-d21e-4892-97ef-c0f45a4d8158"} {"msg":"[2]R flush_numbers[0..12]: [16, 16, 20, 21, 21, 21, 5, 1, 21, 16, 16, 21]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:51.398235294Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1181,"":"downstairs","":"downstairs","session_id":"80107972-d21e-4892-97ef-c0f45a4d8158"} {"msg":"[2]R generation[0..12]: [10, 10, 10, 11, 11, 11, 4, 1, 11, 10, 10, 11]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:51.398245373Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1181,"":"downstairs","":"downstairs","session_id":"80107972-d21e-4892-97ef-c0f45a4d8158"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:51.398256241Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1181,"":"downstairs","":"downstairs","session_id":"80107972-d21e-4892-97ef-c0f45a4d8158"} {"msg":"Max found gen is 12","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:51.3982657Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1181,"":"downstairs","":"downstairs","session_id":"80107972-d21e-4892-97ef-c0f45a4d8158"} {"msg":"Generation requested: 12 >= found:12","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:51.398275888Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1181,"":"downstairs","":"downstairs","session_id":"80107972-d21e-4892-97ef-c0f45a4d8158"} {"msg":"Next flush: 22","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:51.398285177Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1181,"":"downstairs","":"downstairs","session_id":"80107972-d21e-4892-97ef-c0f45a4d8158"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:51.398294575Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1181,"":"downstairs","":"downstairs","session_id":"80107972-d21e-4892-97ef-c0f45a4d8158"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:51.398306004Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1181,"session_id":"80107972-d21e-4892-97ef-c0f45a4d8158"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:51.398314892Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1181,"client":"0","":"downstairs","session_id":"80107972-d21e-4892-97ef-c0f45a4d8158"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:51.398324791Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1181,"client":"1","":"downstairs","session_id":"80107972-d21e-4892-97ef-c0f45a4d8158"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:51.398334669Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1181,"client":"2","":"downstairs","session_id":"80107972-d21e-4892-97ef-c0f45a4d8158"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:51.398344668Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1181,"session_id":"80107972-d21e-4892-97ef-c0f45a4d8158"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:51.398353777Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1181,"session_id":"80107972-d21e-4892-97ef-c0f45a4d8158"} {"msg":"a5d5a495-3110-42b1-92ef-e1f6780ca2d0 is now active with session: 80107972-d21e-4892-97ef-c0f45a4d8158","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:51.398363865Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1181,"session_id":"80107972-d21e-4892-97ef-c0f45a4d8158"} {"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:51.398372734Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1181} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:51.398380283Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1181,"session_id":"80107972-d21e-4892-97ef-c0f45a4d8158"} {"msg":"Activated sub_volume a5d5a495-3110-42b1-92ef-e1f6780ca2d0","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:51.398389401Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1181} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:51.39839749Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1181,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:51.398699946Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1181,"task":"crutest"} 01/30 Write block 284 len 24576 data: 3 3 3 3 2 2 02/30 Write block 356 len 24576 data: 2 2 2 3 3 3 03/30 Read block 319 len 20480 04/30 Read block 276 len 4096 05/30 Write block 351 len 40960 data: 2 2 2 2 2 3 3 3 4 4 06/30 Read block 415 len 36864 07/30 Read block 387 len 36864 08/30 Read block 444 len 36864 09/30 Read block 413 len 20480 10/30 Read block 128 len 32768 11/30 Read block 160 len 20480 12/30 Read block 403 len 16384 13/30 Write block 286 len 32768 data: 4 4 3 3 2 2 2 2 14/30 Read block 151 len 40960 15/30 Read block 518 len 20480 16/30 Read block 246 len 36864 17/30 Write block 552 len 40960 data: 3 3 3 3 3 3 3 2 2 3 18/30 Read block 153 len 8192 19/30 Read block 413 len 28672 20/30 Write block 28 len 20480 data: 3 3 2 2 2 21/30 Read block 110 len 4096 22/30 Read block 183 len 32768 23/30 Write block 138 len 16384 data: 2 2 2 2 24/30 Read block 541 len 28672 25/30 Flush 26/30 Write block 471 len 8192 data: 5 4 27/30 Write block 415 len 16384 data: 3 3 3 2 28/30 Write block 410 len 36864 data: 4 4 4 3 3 4 4 4 3 29/30 Read block 410 len 12288 30/30 Read block 387 len 36864 ---------------------------------------------------------------- Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Sent Done false 1001 Acked Write 6 Done Sent Done false 1002 Acked Read 5 Done Sent Done false 1003 Acked Read 1 Done Sent Done false 1004 Acked Write 10 Done Sent Done false 1005 Acked Read 9 Done Sent Done false 1006 Acked Read 9 Done Sent Done false 1007 Acked Read 9 Done Sent Done false 1008 Acked Read 5 Done Sent Done false 1009 Acked Read 8 Done Sent Done false 1010 Acked Read 5 Done Sent Done false 1011 Acked Read 4 Done Sent Done false 1012 Acked Write 8 Done Sent Done false 1013 Acked Read 10 Done Sent Done false 1014 Acked Read 5 Done Sent Done false 1015 Acked Read 9 Done Sent Done false 1016 Acked Write 10 Done Sent Done false 1017 Acked Read 2 Done Sent Done false 1018 Acked Read 7 Done Sent Done false 1019 Acked Write 5 Done Sent Done false 1020 Acked Read 1 Done Sent Done false 1021 Acked Read 8 Done Sent Done false 1022 Acked Write 4 Done Sent Done false 1023 Acked Read 7 Done Sent Done false 1024 Acked Flush 0 Done Sent Done false 1025 Acked Write 2 Done Sent Done false 1026 Acked Write 4 Done Sent Done false 1027 Acked Write 9 Done Sent Done false 1028 Acked Read 3 Done Sent Done false 1029 Acked Read 9 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 30 0 31 Done 29 0 30 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) None Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:53:51.407758899Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1181} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:51.407935853Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1181,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 16 16 16 F F F 1 020-039 12 10 12 22 16 22 F F F <--- 2 040-059 10 10 10 20 20 20 F F F 3 060-079 11 11 11 21 21 21 F F F 4 080-099 11 11 11 21 21 21 F F F 5 100-119 11 11 11 21 21 21 F F F 6 120-139 12 4 12 22 5 22 F F F <--- 7 140-159 12 1 12 22 1 22 F F F <--- 8 160-179 11 11 11 21 21 21 F F F 9 180-199 10 10 10 16 16 16 F F F 10 200-219 10 10 10 16 16 16 F F F 11 220-239 11 11 11 21 21 21 F F F 12 240-259 11 11 11 21 21 21 F F F 13 260-279 10 10 10 16 16 16 F F F 14 280-299 12 6 12 22 10 22 F F F <--- 15 300-319 3 3 3 3 3 3 F F F 16 320-339 6 6 6 10 10 10 F F F 17 340-359 12 8 12 22 11 22 F F F <--- 18 360-379 12 8 12 22 11 22 F F F <--- 19 380-399 3 3 3 3 3 3 F F F 20 400-419 6 6 6 10 10 10 T F T <--- 21 420-439 10 10 10 16 16 16 F F F 22 440-459 10 10 10 16 16 16 F F F 23 460-479 5 5 5 9 9 9 T F T <--- 24 480-499 6 6 6 10 10 10 F F F 25 500-519 9 9 9 15 15 15 F F F 26 520-539 3 3 3 3 3 3 F F F 27 540-559 12 10 12 22 18 22 F F F <--- 28 560-579 12 4 12 22 6 22 F F F <--- 29 580-599 9 9 9 15 15 15 F F F Max gen: 12, Max flush: 22 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 6 Verifying data now No change after verify Loop: 6 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 16 16 16 F F F 1 020-039 12 12 12 22 22 22 F F F 2 040-059 10 10 10 20 20 20 F F F 3 060-079 11 11 11 21 21 21 F F F 4 080-099 11 11 11 21 21 21 F F F 5 100-119 11 11 11 21 21 21 F F F 6 120-139 12 12 12 22 22 22 F F F 7 140-159 12 12 12 22 22 22 F F F 8 160-179 11 11 11 21 21 21 F F F 9 180-199 10 10 10 16 16 16 F F F 10 200-219 10 10 10 16 16 16 F F F 11 220-239 11 11 11 21 21 21 F F F 12 240-259 11 11 11 21 21 21 F F F 13 260-279 10 10 10 16 16 16 F F F 14 280-299 12 12 12 22 22 22 F F F 15 300-319 3 3 3 3 3 3 F F F 16 320-339 6 6 6 10 10 10 F F F 17 340-359 12 12 12 22 22 22 F F F 18 360-379 12 12 12 22 22 22 F F F 19 380-399 3 3 3 3 3 3 F F F 20 400-419 13 13 13 23 23 23 F F F 21 420-439 10 10 10 16 16 16 F F F 22 440-459 10 10 10 16 16 16 F F F 23 460-479 13 13 13 23 23 23 F F F 24 480-499 6 6 6 10 10 10 F F F 25 500-519 9 9 9 15 15 15 F F F 26 520-539 3 3 3 3 3 3 F F F 27 540-559 12 12 12 22 22 22 F F F 28 560-579 12 12 12 22 22 22 F F F 29 580-599 9 9 9 15 15 15 F F F Max gen: 13, Max flush: 23 Begin loop 7 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:55.647228216Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1189,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:53:55.65445687Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1189,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:55.654519851Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1189,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:55.661673505Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1189,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:55.661957093Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1189} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:55.66198272Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1189} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:55.661993678Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1189} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:55.662005316Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1189} {"msg":"Crucible 31f01ce8-2f12-42b8-ae39-54fb97654bb8 has session id: f6d8763e-a82d-47e1-953f-fb5945e83b96","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:55.662343296Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1189,"session_id":"f6d8763e-a82d-47e1-953f-fb5945e83b96"} {"msg":"Upstairs opts: Upstairs UUID: 31f01ce8-2f12-42b8-ae39-54fb97654bb8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:55.662377491Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1189,"session_id":"f6d8763e-a82d-47e1-953f-fb5945e83b96"} {"msg":"Crucible stats registered with UUID: 31f01ce8-2f12-42b8-ae39-54fb97654bb8","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:55.66238886Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1189,"session_id":"f6d8763e-a82d-47e1-953f-fb5945e83b96"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:53:55.662398548Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1189,"session_id":"f6d8763e-a82d-47e1-953f-fb5945e83b96"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:55.662544507Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1189,"":"io task","client":"2","":"downstairs","session_id":"f6d8763e-a82d-47e1-953f-fb5945e83b96"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:55.662560874Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1189,"":"io task","client":"0","":"downstairs","session_id":"f6d8763e-a82d-47e1-953f-fb5945e83b96"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:55.662571003Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1189,"":"io task","client":"1","":"downstairs","session_id":"f6d8763e-a82d-47e1-953f-fb5945e83b96"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:55.662709562Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1189,"task":"crutest"} {"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:55.66272436Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1189} {"msg":"31f01ce8-2f12-42b8-ae39-54fb97654bb8 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:55.662747807Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1189,"session_id":"f6d8763e-a82d-47e1-953f-fb5945e83b96"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:55.662757325Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1189,"client":"0","":"downstairs","session_id":"f6d8763e-a82d-47e1-953f-fb5945e83b96"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:55.662766984Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1189,"client":"1","":"downstairs","session_id":"f6d8763e-a82d-47e1-953f-fb5945e83b96"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:55.662778862Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1189,"client":"2","":"downstairs","session_id":"f6d8763e-a82d-47e1-953f-fb5945e83b96"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:55.662788511Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1189,"":"io task","client":"2","":"downstairs","session_id":"f6d8763e-a82d-47e1-953f-fb5945e83b96"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:55.662798559Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1189,"":"io task","client":"0","":"downstairs","session_id":"f6d8763e-a82d-47e1-953f-fb5945e83b96"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:55.662808648Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1189,"":"io task","client":"1","":"downstairs","session_id":"f6d8763e-a82d-47e1-953f-fb5945e83b96"} {"msg":"ds_connection connected from Ok(127.0.0.1:37259)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:55.66285955Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1189,"":"io task","client":"2","":"downstairs","session_id":"f6d8763e-a82d-47e1-953f-fb5945e83b96"} {"msg":"ds_connection connected from Ok(127.0.0.1:62234)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:55.662918032Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1189,"":"io task","client":"0","":"downstairs","session_id":"f6d8763e-a82d-47e1-953f-fb5945e83b96"} {"msg":"ds_connection connected from Ok(127.0.0.1:57076)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:55.66292945Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1189,"":"io task","client":"1","":"downstairs","session_id":"f6d8763e-a82d-47e1-953f-fb5945e83b96"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:55.663465471Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1189,"client":"2","":"downstairs","session_id":"f6d8763e-a82d-47e1-953f-fb5945e83b96"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:55.663508504Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1189,"client":"1","":"downstairs","session_id":"f6d8763e-a82d-47e1-953f-fb5945e83b96"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:55.663530301Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1189,"client":"0","":"downstairs","session_id":"f6d8763e-a82d-47e1-953f-fb5945e83b96"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:55.6638104Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1189,"session_id":"f6d8763e-a82d-47e1-953f-fb5945e83b96"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:55.663820778Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1189,"session_id":"f6d8763e-a82d-47e1-953f-fb5945e83b96"} {"msg":"[0]R flush_numbers[0..12]: [16, 22, 20, 21, 21, 21, 22, 22, 21, 16, 16, 21]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:55.663846085Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1189,"":"downstairs","":"downstairs","session_id":"f6d8763e-a82d-47e1-953f-fb5945e83b96"} {"msg":"[0]R generation[0..12]: [10, 12, 10, 11, 11, 11, 12, 12, 11, 10, 10, 11]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:55.663857253Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1189,"":"downstairs","":"downstairs","session_id":"f6d8763e-a82d-47e1-953f-fb5945e83b96"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:55.663868571Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1189,"":"downstairs","":"downstairs","session_id":"f6d8763e-a82d-47e1-953f-fb5945e83b96"} {"msg":"[1]R flush_numbers[0..12]: [16, 22, 20, 21, 21, 21, 22, 22, 21, 16, 16, 21]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:55.663881729Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1189,"":"downstairs","":"downstairs","session_id":"f6d8763e-a82d-47e1-953f-fb5945e83b96"} {"msg":"[1]R generation[0..12]: [10, 12, 10, 11, 11, 11, 12, 12, 11, 10, 10, 11]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:55.663892348Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1189,"":"downstairs","":"downstairs","session_id":"f6d8763e-a82d-47e1-953f-fb5945e83b96"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:55.663903286Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1189,"":"downstairs","":"downstairs","session_id":"f6d8763e-a82d-47e1-953f-fb5945e83b96"} {"msg":"[2]R flush_numbers[0..12]: [16, 22, 20, 21, 21, 21, 22, 22, 21, 16, 16, 21]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:55.663913665Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1189,"":"downstairs","":"downstairs","session_id":"f6d8763e-a82d-47e1-953f-fb5945e83b96"} {"msg":"[2]R generation[0..12]: [10, 12, 10, 11, 11, 11, 12, 12, 11, 10, 10, 11]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:55.663924083Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1189,"":"downstairs","":"downstairs","session_id":"f6d8763e-a82d-47e1-953f-fb5945e83b96"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:55.663934872Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1189,"":"downstairs","":"downstairs","session_id":"f6d8763e-a82d-47e1-953f-fb5945e83b96"} {"msg":"Max found gen is 14","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:55.66394434Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1189,"":"downstairs","":"downstairs","session_id":"f6d8763e-a82d-47e1-953f-fb5945e83b96"} {"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:55.663954329Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1189,"":"downstairs","":"downstairs","session_id":"f6d8763e-a82d-47e1-953f-fb5945e83b96"} {"msg":"Next flush: 24","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:55.663963587Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1189,"":"downstairs","":"downstairs","session_id":"f6d8763e-a82d-47e1-953f-fb5945e83b96"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:55.663973056Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1189,"":"downstairs","":"downstairs","session_id":"f6d8763e-a82d-47e1-953f-fb5945e83b96"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:55.663982874Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1189,"session_id":"f6d8763e-a82d-47e1-953f-fb5945e83b96"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:55.663991663Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1189,"client":"0","":"downstairs","session_id":"f6d8763e-a82d-47e1-953f-fb5945e83b96"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:55.664001382Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1189,"client":"1","":"downstairs","session_id":"f6d8763e-a82d-47e1-953f-fb5945e83b96"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:55.66401129Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1189,"client":"2","":"downstairs","session_id":"f6d8763e-a82d-47e1-953f-fb5945e83b96"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:55.664021309Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1189,"session_id":"f6d8763e-a82d-47e1-953f-fb5945e83b96"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:55.664031917Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1189,"session_id":"f6d8763e-a82d-47e1-953f-fb5945e83b96"} {"msg":"31f01ce8-2f12-42b8-ae39-54fb97654bb8 is now active with session: f6d8763e-a82d-47e1-953f-fb5945e83b96","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:55.664041926Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1189,"session_id":"f6d8763e-a82d-47e1-953f-fb5945e83b96"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:55.664050714Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1189,"session_id":"f6d8763e-a82d-47e1-953f-fb5945e83b96"} {"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:55.664059853Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1189} {"msg":"Activated sub_volume 31f01ce8-2f12-42b8-ae39-54fb97654bb8","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:55.664068122Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1189} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:55.664076391Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1189,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:55.664373617Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1189,"task":"crutest"} 01/30 Write block 12 len 20480 data: 3 3 3 3 3 02/30 Read block 423 len 40960 03/30 Read block 472 len 24576 04/30 Read block 362 len 36864 05/30 Flush 06/30 Write block 246 len 20480 data: 3 2 2 2 2 07/30 Flush 08/30 Flush 09/30 Flush 10/30 Write block 325 len 32768 data: 3 3 3 2 2 3 3 3 11/30 Write block 150 len 20480 data: 2 2 2 2 2 12/30 Write block 141 len 20480 data: 3 2 2 2 2 13/30 Write block 485 len 16384 data: 2 2 3 3 14/30 Write block 574 len 24576 data: 2 2 2 2 2 2 15/30 Read block 341 len 28672 16/30 Read block 430 len 8192 17/30 Read block 99 len 36864 18/30 Write block 591 len 28672 data: 3 3 4 4 3 2 2 19/30 Read block 108 len 12288 20/30 Write block 240 len 36864 data: 4 4 4 4 4 4 4 3 3 21/30 Flush 22/30 Flush 23/30 Write block 37 len 40960 data: 4 4 4 4 4 3 4 3 2 2 24/30 Write block 592 len 8192 data: 4 5 25/30 Write block 157 len 8192 data: 2 2 26/30 Read block 520 len 40960 27/30 Read block 284 len 40960 28/30 Read block 213 len 36864 29/30 Read block 329 len 8192 30/30 Read block 287 len 36864 ---------------------------------------------------------------- Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Read 10 Done Done Done false 1002 Acked Read 6 Done Done Done false 1003 Acked Read 9 Done Done Done false 1004 Acked Flush 0 Sent Done Done false 1005 Acked Write 5 Sent Done Done false 1006 Acked Flush 0 Sent Done Done false 1007 Acked Flush 0 Sent Done Done false 1008 Acked Flush 0 Sent Done Done false 1009 Acked Write 8 Sent Done Done false 1010 Acked Write 5 Sent Done Done false 1011 Acked Write 5 Sent Done Done false 1012 Acked Write 4 Sent Done Done false 1013 Acked Write 6 Sent Done Done false 1014 Acked Read 7 Sent Done Done false 1015 Acked Read 2 Sent Done Done false 1016 Acked Read 9 Sent Done Done false 1017 Acked Write 7 Sent Done Done false 1018 Acked Read 3 Sent Done Done false 1019 Acked Write 9 Sent Done Done false 1020 Acked Flush 0 Sent Done Done false 1021 Acked Flush 0 Sent Done Done false 1022 Acked Write 10 Sent Done Done false 1023 Acked Write 2 Sent Done Done false 1024 Acked Write 2 Sent Done Done false 1025 Acked Read 10 Sent Done Done false 1026 Acked Read 10 Sent Done Done false 1027 Acked Read 9 Sent Done Done false 1028 Acked Read 2 Sent Done Done false 1029 Acked Read 9 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 26 1 0 27 Done 4 29 30 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:53:55.673761012Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1189} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:55.673923598Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1189,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 14 14 16 24 24 T F F <--- 1 020-039 12 12 12 22 22 22 F T T <--- 2 040-059 10 10 10 20 20 20 F T T <--- 3 060-079 11 11 11 21 21 21 F F F 4 080-099 11 11 11 21 21 21 F F F 5 100-119 11 11 11 21 21 21 F F F 6 120-139 12 12 12 22 22 22 F F F 7 140-159 12 14 14 22 28 28 F T T <--- 8 160-179 11 11 11 21 21 21 F F F 9 180-199 10 10 10 16 16 16 F F F 10 200-219 10 10 10 16 16 16 F F F 11 220-239 11 11 11 21 21 21 F F F 12 240-259 11 14 14 21 28 28 F F F <--- 13 260-279 10 10 10 16 16 16 F F F 14 280-299 12 12 12 22 22 22 F F F 15 300-319 3 3 3 3 3 3 F F F 16 320-339 6 14 14 10 28 28 F F F <--- 17 340-359 12 12 12 22 22 22 F F F 18 360-379 12 12 12 22 22 22 F F F 19 380-399 3 3 3 3 3 3 F F F 20 400-419 13 13 13 23 23 23 F F F 21 420-439 10 10 10 16 16 16 F F F 22 440-459 10 10 10 16 16 16 F F F 23 460-479 13 13 13 23 23 23 F F F 24 480-499 6 14 14 10 28 28 F F F <--- 25 500-519 9 9 9 15 15 15 F F F 26 520-539 3 3 3 3 3 3 F F F 27 540-559 12 12 12 22 22 22 F F F 28 560-579 12 14 14 22 28 28 F F F <--- 29 580-599 9 14 14 15 28 28 F T T <--- Max gen: 14, Max flush: 28 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 7 Verifying data now No change after verify Loop: 7 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 14 14 14 24 24 24 F F F 1 020-039 15 15 15 29 29 29 F F F 2 040-059 15 15 15 29 29 29 F F F 3 060-079 11 11 11 21 21 21 F F F 4 080-099 11 11 11 21 21 21 F F F 5 100-119 11 11 11 21 21 21 F F F 6 120-139 12 12 12 22 22 22 F F F 7 140-159 15 15 15 29 29 29 F F F 8 160-179 11 11 11 21 21 21 F F F 9 180-199 10 10 10 16 16 16 F F F 10 200-219 10 10 10 16 16 16 F F F 11 220-239 11 11 11 21 21 21 F F F 12 240-259 14 14 14 28 28 28 F F F 13 260-279 10 10 10 16 16 16 F F F 14 280-299 12 12 12 22 22 22 F F F 15 300-319 3 3 3 3 3 3 F F F 16 320-339 14 14 14 28 28 28 F F F 17 340-359 12 12 12 22 22 22 F F F 18 360-379 12 12 12 22 22 22 F F F 19 380-399 3 3 3 3 3 3 F F F 20 400-419 13 13 13 23 23 23 F F F 21 420-439 10 10 10 16 16 16 F F F 22 440-459 10 10 10 16 16 16 F F F 23 460-479 13 13 13 23 23 23 F F F 24 480-499 14 14 14 28 28 28 F F F 25 500-519 9 9 9 15 15 15 F F F 26 520-539 3 3 3 3 3 3 F F F 27 540-559 12 12 12 22 22 22 F F F 28 560-579 14 14 14 28 28 28 F F F 29 580-599 15 15 15 29 29 29 F F F Max gen: 15, Max flush: 29 Begin loop 8 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:59.920765007Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1197,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:59.929067262Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1197,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:59.929375916Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1197} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:59.929402063Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1197} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:59.929410811Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1197} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:59.92942239Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1197} {"msg":"Crucible 5cb5240e-c33f-4da1-8a94-160cadc1193e has session id: 29deacc2-c229-4a29-9f5b-20883b826125","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:59.929779057Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1197,"session_id":"29deacc2-c229-4a29-9f5b-20883b826125"} {"msg":"Upstairs opts: Upstairs UUID: 5cb5240e-c33f-4da1-8a94-160cadc1193e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:59.929804563Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1197,"session_id":"29deacc2-c229-4a29-9f5b-20883b826125"} {"msg":"Crucible stats registered with UUID: 5cb5240e-c33f-4da1-8a94-160cadc1193e","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:59.929814712Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1197,"session_id":"29deacc2-c229-4a29-9f5b-20883b826125"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:53:59.92982643Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1197,"session_id":"29deacc2-c229-4a29-9f5b-20883b826125"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:59.929984347Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1197,"":"io task","client":"0","":"downstairs","session_id":"29deacc2-c229-4a29-9f5b-20883b826125"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:59.93002643Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1197,"":"io task","client":"2","":"downstairs","session_id":"29deacc2-c229-4a29-9f5b-20883b826125"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:59.930037919Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1197,"":"io task","client":"1","":"downstairs","session_id":"29deacc2-c229-4a29-9f5b-20883b826125"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:59.930135974Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1197,"task":"crutest"} {"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:59.930154811Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1197} {"msg":"5cb5240e-c33f-4da1-8a94-160cadc1193e active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:59.930172549Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1197,"session_id":"29deacc2-c229-4a29-9f5b-20883b826125"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:59.930181408Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1197,"client":"0","":"downstairs","session_id":"29deacc2-c229-4a29-9f5b-20883b826125"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:59.930191226Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1197,"client":"1","":"downstairs","session_id":"29deacc2-c229-4a29-9f5b-20883b826125"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:59.930200955Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1197,"client":"2","":"downstairs","session_id":"29deacc2-c229-4a29-9f5b-20883b826125"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:59.930210413Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1197,"":"io task","client":"2","":"downstairs","session_id":"29deacc2-c229-4a29-9f5b-20883b826125"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:59.930220572Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1197,"":"io task","client":"0","":"downstairs","session_id":"29deacc2-c229-4a29-9f5b-20883b826125"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:59.93023074Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1197,"":"io task","client":"1","":"downstairs","session_id":"29deacc2-c229-4a29-9f5b-20883b826125"} {"msg":"ds_connection connected from Ok(127.0.0.1:47347)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:59.930316938Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1197,"":"io task","client":"2","":"downstairs","session_id":"29deacc2-c229-4a29-9f5b-20883b826125"} {"msg":"ds_connection connected from Ok(127.0.0.1:62250)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:59.930342724Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1197,"":"io task","client":"0","":"downstairs","session_id":"29deacc2-c229-4a29-9f5b-20883b826125"} {"msg":"ds_connection connected from Ok(127.0.0.1:52779)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:59.930354362Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1197,"":"io task","client":"1","":"downstairs","session_id":"29deacc2-c229-4a29-9f5b-20883b826125"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:59.931009845Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1197,"client":"1","":"downstairs","session_id":"29deacc2-c229-4a29-9f5b-20883b826125"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:59.931028743Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1197,"client":"0","":"downstairs","session_id":"29deacc2-c229-4a29-9f5b-20883b826125"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:59.931039851Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1197,"client":"2","":"downstairs","session_id":"29deacc2-c229-4a29-9f5b-20883b826125"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:59.931338747Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1197,"session_id":"29deacc2-c229-4a29-9f5b-20883b826125"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:59.931362753Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1197,"session_id":"29deacc2-c229-4a29-9f5b-20883b826125"} {"msg":"[0]R flush_numbers[0..12]: [24, 29, 29, 21, 21, 21, 22, 29, 21, 16, 16, 21]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:59.931418135Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1197,"":"downstairs","":"downstairs","session_id":"29deacc2-c229-4a29-9f5b-20883b826125"} {"msg":"[0]R generation[0..12]: [14, 15, 15, 11, 11, 11, 12, 15, 11, 10, 10, 11]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:59.931430033Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1197,"":"downstairs","":"downstairs","session_id":"29deacc2-c229-4a29-9f5b-20883b826125"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:59.931441252Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1197,"":"downstairs","":"downstairs","session_id":"29deacc2-c229-4a29-9f5b-20883b826125"} {"msg":"[1]R flush_numbers[0..12]: [24, 29, 29, 21, 21, 21, 22, 29, 21, 16, 16, 21]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:59.93145168Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1197,"":"downstairs","":"downstairs","session_id":"29deacc2-c229-4a29-9f5b-20883b826125"} {"msg":"[1]R generation[0..12]: [14, 15, 15, 11, 11, 11, 12, 15, 11, 10, 10, 11]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:59.931462429Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1197,"":"downstairs","":"downstairs","session_id":"29deacc2-c229-4a29-9f5b-20883b826125"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:59.931473317Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1197,"":"downstairs","":"downstairs","session_id":"29deacc2-c229-4a29-9f5b-20883b826125"} {"msg":"[2]R flush_numbers[0..12]: [24, 29, 29, 21, 21, 21, 22, 29, 21, 16, 16, 21]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:59.931483885Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1197,"":"downstairs","":"downstairs","session_id":"29deacc2-c229-4a29-9f5b-20883b826125"} {"msg":"[2]R generation[0..12]: [14, 15, 15, 11, 11, 11, 12, 15, 11, 10, 10, 11]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:59.931494234Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1197,"":"downstairs","":"downstairs","session_id":"29deacc2-c229-4a29-9f5b-20883b826125"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:59.931505392Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1197,"":"downstairs","":"downstairs","session_id":"29deacc2-c229-4a29-9f5b-20883b826125"} {"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:59.9315176Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1197,"":"downstairs","":"downstairs","session_id":"29deacc2-c229-4a29-9f5b-20883b826125"} {"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:59.931527749Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1197,"":"downstairs","":"downstairs","session_id":"29deacc2-c229-4a29-9f5b-20883b826125"} {"msg":"Next flush: 30","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:59.931537268Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1197,"":"downstairs","":"downstairs","session_id":"29deacc2-c229-4a29-9f5b-20883b826125"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:59.931546826Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1197,"":"downstairs","":"downstairs","session_id":"29deacc2-c229-4a29-9f5b-20883b826125"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:59.931556695Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1197,"session_id":"29deacc2-c229-4a29-9f5b-20883b826125"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:59.931565503Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1197,"client":"0","":"downstairs","session_id":"29deacc2-c229-4a29-9f5b-20883b826125"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:59.931575162Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1197,"client":"1","":"downstairs","session_id":"29deacc2-c229-4a29-9f5b-20883b826125"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:59.931584771Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1197,"client":"2","":"downstairs","session_id":"29deacc2-c229-4a29-9f5b-20883b826125"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:59.931594649Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1197,"session_id":"29deacc2-c229-4a29-9f5b-20883b826125"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:59.931603768Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1197,"session_id":"29deacc2-c229-4a29-9f5b-20883b826125"} {"msg":"5cb5240e-c33f-4da1-8a94-160cadc1193e is now active with session: 29deacc2-c229-4a29-9f5b-20883b826125","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:59.931613706Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1197,"session_id":"29deacc2-c229-4a29-9f5b-20883b826125"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:59.931622445Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1197,"session_id":"29deacc2-c229-4a29-9f5b-20883b826125"} {"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:59.931631624Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1197} {"msg":"Activated sub_volume 5cb5240e-c33f-4da1-8a94-160cadc1193e","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:59.931639852Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1197} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:59.931648001Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1197,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:59.931973503Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1197,"task":"crutest"} 01/30 Write block 312 len 24576 data: 2 2 2 2 2 2 02/30 Write block 311 len 4096 data: 2 03/30 Write block 275 len 24576 data: 3 3 3 3 2 2 04/30 Read block 552 len 20480 05/30 Write block 510 len 12288 data: 3 2 2 06/30 Write block 416 len 40960 data: 5 5 4 2 2 2 2 2 2 2 07/30 Write block 543 len 16384 data: 2 2 3 2 08/30 Read block 181 len 28672 09/30 Write block 156 len 32768 data: 2 3 3 2 2 2 2 2 10/30 Read block 494 len 12288 11/30 Write block 263 len 40960 data: 4 3 3 4 4 2 2 2 3 3 12/30 Write block 573 len 36864 data: 2 3 3 3 3 3 3 2 3 13/30 Read block 531 len 16384 14/30 Write block 33 len 32768 data: 2 3 4 4 5 5 5 5 15/30 Write block 441 len 28672 data: 2 2 2 3 3 3 2 16/30 Flush 17/30 Write block 444 len 40960 data: 4 4 4 3 3 3 4 4 4 2 18/30 Read block 157 len 16384 19/30 Write block 305 len 36864 data: 2 2 2 2 2 2 3 3 3 20/30 Write block 62 len 4096 data: 2 21/30 Write block 191 len 16384 data: 2 2 2 2 22/30 Read block 282 len 8192 23/30 Read block 353 len 16384 24/30 Write block 457 len 12288 data: 2 2 2 25/30 Write block 517 len 12288 data: 3 4 3 26/30 Read block 489 len 12288 27/30 Read block 210 len 20480 28/30 Write block 257 len 12288 data: 2 3 3 29/30 Write block 390 len 12288 data: 4 4 4 30/30 Read block 295 len 40960 ---------------------------------------------------------------- Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Write 1 Done Done Sent false 1002 Acked Write 6 Done Done Sent false 1003 Acked Read 5 Done Done Sent false 1004 Acked Write 3 Done Done Sent false 1005 Acked Write 10 Done Done Sent false 1006 Acked Write 4 Done Done Sent false 1007 Acked Read 7 Done Done Sent false 1008 Acked Write 8 Done Done Sent false 1009 Acked Read 3 Done Done Sent false 1010 Acked Write 10 Done Done Sent false 1011 Acked Write 9 Done Done Sent false 1012 Acked Read 4 Done Done Sent false 1013 Acked Write 8 Done Done Sent false 1014 Acked Write 7 Done Done Sent false 1015 Acked Flush 0 Done Done Sent false 1016 Acked Write 10 Done Done Sent false 1017 Acked Read 4 Done Done Sent false 1018 Acked Write 9 Done Done Sent false 1019 Acked Write 1 Done Done Sent false 1020 Acked Write 4 Done Done Sent false 1021 Acked Read 2 Done Done Sent false 1022 Acked Read 4 Done Done Sent false 1023 Acked Write 3 Done Done Sent false 1024 Acked Write 3 Done Done Sent false 1025 Acked Read 3 Done Done Sent false 1026 Acked Read 5 Done Done Sent false 1027 Acked Write 3 Done Done Sent false 1028 Acked Write 3 Done Done Sent false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 29 30 Done 30 29 1 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) Some(JobId(1015)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:53:59.940168594Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1197} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:53:59.940345388Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1197,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 14 14 14 24 24 24 F F F 1 020-039 16 16 15 30 30 29 F F F <--- 2 040-059 16 16 15 30 30 29 F F F <--- 3 060-079 11 11 11 21 21 21 T T F <--- 4 080-099 11 11 11 21 21 21 F F F 5 100-119 11 11 11 21 21 21 F F F 6 120-139 12 12 12 22 22 22 F F F 7 140-159 16 16 15 30 30 29 F F F <--- 8 160-179 16 16 11 30 30 21 F F F <--- 9 180-199 10 10 10 16 16 16 T T F <--- 10 200-219 10 10 10 16 16 16 F F F 11 220-239 11 11 11 21 21 21 F F F 12 240-259 14 14 14 28 28 28 T T F <--- 13 260-279 16 16 10 30 30 16 F F F <--- 14 280-299 16 16 12 30 30 22 F F F <--- 15 300-319 16 16 3 30 30 3 T T T <--- 16 320-339 14 14 14 28 28 28 F F F 17 340-359 12 12 12 22 22 22 F F F 18 360-379 12 12 12 22 22 22 F F F 19 380-399 3 3 3 3 3 3 T T F <--- 20 400-419 16 16 13 30 30 23 F F F <--- 21 420-439 16 16 10 30 30 16 F F F <--- 22 440-459 16 16 10 30 30 16 T T F <--- 23 460-479 13 13 13 23 23 23 F F F 24 480-499 14 14 14 28 28 28 F F F 25 500-519 16 16 9 30 30 15 T T F <--- 26 520-539 3 3 3 3 3 3 F F F 27 540-559 16 16 12 30 30 22 F F F <--- 28 560-579 16 16 14 30 30 28 F F F <--- 29 580-599 16 16 15 30 30 29 F F F <--- Max gen: 16, Max flush: 30 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 8 Verifying data now No change after verify Loop: 8 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 14 14 14 24 24 24 F F F 1 020-039 16 16 16 30 30 30 F F F 2 040-059 16 16 16 30 30 30 F F F 3 060-079 17 17 17 31 31 31 F F F 4 080-099 11 11 11 21 21 21 F F F 5 100-119 11 11 11 21 21 21 F F F 6 120-139 12 12 12 22 22 22 F F F 7 140-159 16 16 16 30 30 30 F F F 8 160-179 16 16 16 30 30 30 F F F 9 180-199 17 17 17 31 31 31 F F F 10 200-219 10 10 10 16 16 16 F F F 11 220-239 11 11 11 21 21 21 F F F 12 240-259 17 17 17 31 31 31 F F F 13 260-279 16 16 16 30 30 30 F F F 14 280-299 16 16 16 30 30 30 F F F 15 300-319 17 17 17 31 31 31 F F F 16 320-339 14 14 14 28 28 28 F F F 17 340-359 12 12 12 22 22 22 F F F 18 360-379 12 12 12 22 22 22 F F F 19 380-399 17 17 17 31 31 31 F F F 20 400-419 16 16 16 30 30 30 F F F 21 420-439 16 16 16 30 30 30 F F F 22 440-459 17 17 17 31 31 31 F F F 23 460-479 13 13 13 23 23 23 F F F 24 480-499 14 14 14 28 28 28 F F F 25 500-519 17 17 17 31 31 31 F F F 26 520-539 3 3 3 3 3 3 F F F 27 540-559 16 16 16 30 30 30 F F F 28 560-579 16 16 16 30 30 30 F F F 29 580-599 16 16 16 30 30 30 F F F Max gen: 17, Max flush: 31 Begin loop 9 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:04.195904481Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1205,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:04.204199518Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1205,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:04.204501053Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1205} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:04.204529119Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1205} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:04.204537488Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1205} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:04.204549096Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1205} {"msg":"Crucible cb4b0119-a5ab-4da7-ab4e-ba7f0b5e2014 has session id: eb1d06ca-ee9f-4478-b6f3-cd84552bcd87","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:04.204868689Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1205,"session_id":"eb1d06ca-ee9f-4478-b6f3-cd84552bcd87"} {"msg":"Upstairs opts: Upstairs UUID: cb4b0119-a5ab-4da7-ab4e-ba7f0b5e2014, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:04.204897465Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1205,"session_id":"eb1d06ca-ee9f-4478-b6f3-cd84552bcd87"} {"msg":"Crucible stats registered with UUID: cb4b0119-a5ab-4da7-ab4e-ba7f0b5e2014","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:04.204908543Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1205,"session_id":"eb1d06ca-ee9f-4478-b6f3-cd84552bcd87"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:54:04.204917722Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1205,"session_id":"eb1d06ca-ee9f-4478-b6f3-cd84552bcd87"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:04.205038614Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1205,"":"io task","client":"1","":"downstairs","session_id":"eb1d06ca-ee9f-4478-b6f3-cd84552bcd87"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:04.205055391Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1205,"":"io task","client":"0","":"downstairs","session_id":"eb1d06ca-ee9f-4478-b6f3-cd84552bcd87"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:04.20506533Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1205,"":"io task","client":"2","":"downstairs","session_id":"eb1d06ca-ee9f-4478-b6f3-cd84552bcd87"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:04.205219137Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1205,"task":"crutest"} {"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:04.205243624Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1205} {"msg":"cb4b0119-a5ab-4da7-ab4e-ba7f0b5e2014 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:04.205254322Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1205,"session_id":"eb1d06ca-ee9f-4478-b6f3-cd84552bcd87"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:04.205263181Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1205,"client":"0","":"downstairs","session_id":"eb1d06ca-ee9f-4478-b6f3-cd84552bcd87"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:04.205272839Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1205,"client":"1","":"downstairs","session_id":"eb1d06ca-ee9f-4478-b6f3-cd84552bcd87"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:04.205282338Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1205,"client":"2","":"downstairs","session_id":"eb1d06ca-ee9f-4478-b6f3-cd84552bcd87"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:04.205291747Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1205,"":"io task","client":"2","":"downstairs","session_id":"eb1d06ca-ee9f-4478-b6f3-cd84552bcd87"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:04.205301825Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1205,"":"io task","client":"0","":"downstairs","session_id":"eb1d06ca-ee9f-4478-b6f3-cd84552bcd87"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:04.205311814Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1205,"":"io task","client":"1","":"downstairs","session_id":"eb1d06ca-ee9f-4478-b6f3-cd84552bcd87"} {"msg":"ds_connection connected from Ok(127.0.0.1:64046)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:04.205346468Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1205,"":"io task","client":"0","":"downstairs","session_id":"eb1d06ca-ee9f-4478-b6f3-cd84552bcd87"} {"msg":"ds_connection connected from Ok(127.0.0.1:52296)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:04.205357617Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1205,"":"io task","client":"1","":"downstairs","session_id":"eb1d06ca-ee9f-4478-b6f3-cd84552bcd87"} {"msg":"ds_connection connected from Ok(127.0.0.1:54281)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:04.205436265Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1205,"":"io task","client":"2","":"downstairs","session_id":"eb1d06ca-ee9f-4478-b6f3-cd84552bcd87"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:04.206019889Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1205,"client":"0","":"downstairs","session_id":"eb1d06ca-ee9f-4478-b6f3-cd84552bcd87"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:04.206040176Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1205,"client":"1","":"downstairs","session_id":"eb1d06ca-ee9f-4478-b6f3-cd84552bcd87"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:04.206051254Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1205,"client":"2","":"downstairs","session_id":"eb1d06ca-ee9f-4478-b6f3-cd84552bcd87"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:04.206301787Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1205,"session_id":"eb1d06ca-ee9f-4478-b6f3-cd84552bcd87"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:04.206338542Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1205,"session_id":"eb1d06ca-ee9f-4478-b6f3-cd84552bcd87"} {"msg":"[0]R flush_numbers[0..12]: [24, 30, 30, 31, 21, 21, 22, 30, 30, 31, 16, 21]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:04.206386425Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1205,"":"downstairs","":"downstairs","session_id":"eb1d06ca-ee9f-4478-b6f3-cd84552bcd87"} {"msg":"[0]R generation[0..12]: [14, 16, 16, 17, 11, 11, 12, 16, 16, 17, 10, 11]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:04.206397703Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1205,"":"downstairs","":"downstairs","session_id":"eb1d06ca-ee9f-4478-b6f3-cd84552bcd87"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:04.206408652Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1205,"":"downstairs","":"downstairs","session_id":"eb1d06ca-ee9f-4478-b6f3-cd84552bcd87"} {"msg":"[1]R flush_numbers[0..12]: [24, 30, 30, 31, 21, 21, 22, 30, 30, 31, 16, 21]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:04.20641903Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1205,"":"downstairs","":"downstairs","session_id":"eb1d06ca-ee9f-4478-b6f3-cd84552bcd87"} {"msg":"[1]R generation[0..12]: [14, 16, 16, 17, 11, 11, 12, 16, 16, 17, 10, 11]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:04.206429579Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1205,"":"downstairs","":"downstairs","session_id":"eb1d06ca-ee9f-4478-b6f3-cd84552bcd87"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:04.206440577Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1205,"":"downstairs","":"downstairs","session_id":"eb1d06ca-ee9f-4478-b6f3-cd84552bcd87"} {"msg":"[2]R flush_numbers[0..12]: [24, 30, 30, 31, 21, 21, 22, 30, 30, 31, 16, 21]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:04.206450985Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1205,"":"downstairs","":"downstairs","session_id":"eb1d06ca-ee9f-4478-b6f3-cd84552bcd87"} {"msg":"[2]R generation[0..12]: [14, 16, 16, 17, 11, 11, 12, 16, 16, 17, 10, 11]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:04.206461324Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1205,"":"downstairs","":"downstairs","session_id":"eb1d06ca-ee9f-4478-b6f3-cd84552bcd87"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:04.206472292Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1205,"":"downstairs","":"downstairs","session_id":"eb1d06ca-ee9f-4478-b6f3-cd84552bcd87"} {"msg":"Max found gen is 18","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:04.206481841Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1205,"":"downstairs","":"downstairs","session_id":"eb1d06ca-ee9f-4478-b6f3-cd84552bcd87"} {"msg":"Generation requested: 18 >= found:18","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:04.206491489Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1205,"":"downstairs","":"downstairs","session_id":"eb1d06ca-ee9f-4478-b6f3-cd84552bcd87"} {"msg":"Next flush: 32","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:04.206500948Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1205,"":"downstairs","":"downstairs","session_id":"eb1d06ca-ee9f-4478-b6f3-cd84552bcd87"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:04.206510397Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1205,"":"downstairs","":"downstairs","session_id":"eb1d06ca-ee9f-4478-b6f3-cd84552bcd87"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:04.206520115Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1205,"session_id":"eb1d06ca-ee9f-4478-b6f3-cd84552bcd87"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:04.206529064Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1205,"client":"0","":"downstairs","session_id":"eb1d06ca-ee9f-4478-b6f3-cd84552bcd87"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:04.206540542Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1205,"client":"1","":"downstairs","session_id":"eb1d06ca-ee9f-4478-b6f3-cd84552bcd87"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:04.206550381Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1205,"client":"2","":"downstairs","session_id":"eb1d06ca-ee9f-4478-b6f3-cd84552bcd87"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:04.206560169Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1205,"session_id":"eb1d06ca-ee9f-4478-b6f3-cd84552bcd87"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:04.206569388Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1205,"session_id":"eb1d06ca-ee9f-4478-b6f3-cd84552bcd87"} {"msg":"cb4b0119-a5ab-4da7-ab4e-ba7f0b5e2014 is now active with session: eb1d06ca-ee9f-4478-b6f3-cd84552bcd87","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:04.206579317Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1205,"session_id":"eb1d06ca-ee9f-4478-b6f3-cd84552bcd87"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:04.206588025Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1205,"session_id":"eb1d06ca-ee9f-4478-b6f3-cd84552bcd87"} {"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:04.206597234Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1205} {"msg":"Activated sub_volume cb4b0119-a5ab-4da7-ab4e-ba7f0b5e2014","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:04.206605623Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1205} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:04.206613891Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1205,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:04.206955391Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1205,"task":"crutest"} 01/30 Write block 144 len 24576 data: 3 3 2 2 2 2 02/30 Flush 03/30 Write block 3 len 24576 data: 2 2 2 2 2 2 04/30 Flush 05/30 Write block 444 len 20480 data: 5 5 5 4 4 06/30 Read block 390 len 28672 07/30 Write block 220 len 36864 data: 2 2 2 2 3 2 2 2 2 08/30 Read block 235 len 12288 09/30 Write block 41 len 32768 data: 5 4 5 4 3 3 3 3 10/30 Write block 266 len 8192 data: 5 5 11/30 Flush 12/30 Write block 67 len 8192 data: 2 2 13/30 Write block 550 len 36864 data: 2 2 4 4 4 4 4 4 4 14/30 Write block 152 len 24576 data: 3 3 3 2 3 4 15/30 Flush 16/30 Write block 482 len 28672 data: 4 3 3 3 3 4 4 17/30 Read block 578 len 24576 18/30 Flush 19/30 Write block 202 len 24576 data: 3 3 3 3 3 4 20/30 Read block 376 len 40960 21/30 Read block 151 len 12288 22/30 Flush 23/30 Write block 581 len 24576 data: 4 3 3 4 4 3 24/30 Write block 451 len 16384 data: 5 5 3 2 25/30 Write block 350 len 16384 data: 2 3 3 3 26/30 Read block 218 len 24576 27/30 Write block 395 len 12288 data: 3 2 2 28/30 Write block 143 len 36864 data: 3 4 4 3 3 3 3 3 3 29/30 Read block 46 len 36864 30/30 Read block 336 len 4096 ---------------------------------------------------------------- Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 6 Done Done Sent false 1003 Acked Flush 0 Done Done Sent false 1004 Acked Write 5 Done Done Sent false 1005 Acked Read 7 Done Done Sent false 1006 Acked Write 9 Done Done Sent false 1007 Acked Read 3 Done Done Sent false 1008 Acked Write 8 Done Done Sent false 1009 Acked Write 2 Done Done Sent false 1010 Acked Flush 0 Done Done Sent false 1011 Acked Write 2 Done Done Sent false 1012 Acked Write 9 Done Done Sent false 1013 Acked Write 6 Done Done Sent false 1014 Acked Flush 0 Done Done Sent false 1015 Acked Write 7 Done Done Sent false 1016 Acked Read 6 Done Done Sent false 1017 Acked Flush 0 Done Done Sent false 1018 Acked Write 6 Done Done Sent false 1019 Acked Read 10 Done Done Sent false 1020 Acked Read 3 Done Done Sent false 1021 Acked Flush 0 Done Done Sent false 1022 Acked Write 6 Done Done Sent false 1023 Acked Write 4 Done Done Sent false 1024 Acked Write 4 Done Done Sent false 1025 Acked Read 6 Done Done Sent false 1026 Acked Write 3 Done Done Sent false 1027 Acked Write 9 Done Done Sent false 1028 Acked Read 9 Done Done Sent false 1029 Acked Read 1 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 28 29 Done 27 28 0 55 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1001)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:54:04.21570461Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1205} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:04.215899611Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1205,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 18 18 14 33 33 24 F F F <--- 1 020-039 16 16 16 30 30 30 F F F 2 040-059 18 18 16 34 34 30 F F F <--- 3 060-079 18 18 17 35 35 31 F F F <--- 4 080-099 11 11 11 21 21 21 F F F 5 100-119 11 11 11 21 21 21 F F F 6 120-139 12 12 12 22 22 22 F F F 7 140-159 18 18 18 35 35 32 T T F <--- 8 160-179 16 16 16 30 30 30 F F F 9 180-199 17 17 17 31 31 31 F F F 10 200-219 18 18 10 37 37 16 F F F <--- 11 220-239 18 18 11 34 34 21 F F F <--- 12 240-259 17 17 17 31 31 31 F F F 13 260-279 18 18 16 34 34 30 F F F <--- 14 280-299 16 16 16 30 30 30 F F F 15 300-319 17 17 17 31 31 31 F F F 16 320-339 14 14 14 28 28 28 F F F 17 340-359 12 12 12 22 22 22 T T F <--- 18 360-379 12 12 12 22 22 22 F F F 19 380-399 17 17 17 31 31 31 T T F <--- 20 400-419 16 16 16 30 30 30 F F F 21 420-439 16 16 16 30 30 30 F F F 22 440-459 18 18 17 34 34 31 T T F <--- 23 460-479 13 13 13 23 23 23 F F F 24 480-499 18 18 14 36 36 28 F F F <--- 25 500-519 17 17 17 31 31 31 F F F 26 520-539 3 3 3 3 3 3 F F F 27 540-559 18 18 16 35 35 30 F F F <--- 28 560-579 16 16 16 30 30 30 F F F 29 580-599 16 16 16 30 30 30 T T F <--- Max gen: 18, Max flush: 37 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 9 Verifying data now No change after verify Loop: 9 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 18 18 18 33 33 33 F F F 1 020-039 16 16 16 30 30 30 F F F 2 040-059 18 18 18 34 34 34 F F F 3 060-079 18 18 18 35 35 35 F F F 4 080-099 11 11 11 21 21 21 F F F 5 100-119 11 11 11 21 21 21 F F F 6 120-139 12 12 12 22 22 22 F F F 7 140-159 19 19 19 38 38 38 F F F 8 160-179 16 16 16 30 30 30 F F F 9 180-199 17 17 17 31 31 31 F F F 10 200-219 18 18 18 37 37 37 F F F 11 220-239 18 18 18 34 34 34 F F F 12 240-259 17 17 17 31 31 31 F F F 13 260-279 18 18 18 34 34 34 F F F 14 280-299 16 16 16 30 30 30 F F F 15 300-319 17 17 17 31 31 31 F F F 16 320-339 14 14 14 28 28 28 F F F 17 340-359 19 19 19 38 38 38 F F F 18 360-379 12 12 12 22 22 22 F F F 19 380-399 19 19 19 38 38 38 F F F 20 400-419 16 16 16 30 30 30 F F F 21 420-439 16 16 16 30 30 30 F F F 22 440-459 19 19 19 38 38 38 F F F 23 460-479 13 13 13 23 23 23 F F F 24 480-499 18 18 18 36 36 36 F F F 25 500-519 17 17 17 31 31 31 F F F 26 520-539 3 3 3 3 3 3 F F F 27 540-559 18 18 18 35 35 35 F F F 28 560-579 16 16 16 30 30 30 F F F 29 580-599 19 19 19 38 38 38 F F F Max gen: 19, Max flush: 38 Begin loop 10 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:08.459998146Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1213,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:54:08.467095179Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1213,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:08.467150211Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1213,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:08.473876598Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1213,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:08.47413766Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1213} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:08.474162976Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1213} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:08.474171515Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1213} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:08.474185513Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1213} {"msg":"Crucible 11f48810-1f1c-4140-b10e-b0bb0a75b8e3 has session id: b3dcf985-4eba-44cd-900b-fa9a3d49ab09","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:08.474579815Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1213,"session_id":"b3dcf985-4eba-44cd-900b-fa9a3d49ab09"} {"msg":"Upstairs opts: Upstairs UUID: 11f48810-1f1c-4140-b10e-b0bb0a75b8e3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:08.4746091Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1213,"session_id":"b3dcf985-4eba-44cd-900b-fa9a3d49ab09"} {"msg":"Crucible stats registered with UUID: 11f48810-1f1c-4140-b10e-b0bb0a75b8e3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:08.474620539Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1213,"session_id":"b3dcf985-4eba-44cd-900b-fa9a3d49ab09"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:54:08.474629717Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1213,"session_id":"b3dcf985-4eba-44cd-900b-fa9a3d49ab09"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:08.474757358Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1213,"":"io task","client":"1","":"downstairs","session_id":"b3dcf985-4eba-44cd-900b-fa9a3d49ab09"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:08.474788524Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1213,"":"io task","client":"0","":"downstairs","session_id":"b3dcf985-4eba-44cd-900b-fa9a3d49ab09"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:08.474799562Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1213,"":"io task","client":"2","":"downstairs","session_id":"b3dcf985-4eba-44cd-900b-fa9a3d49ab09"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:08.474924524Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1213,"task":"crutest"} {"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:08.474938302Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1213} {"msg":"11f48810-1f1c-4140-b10e-b0bb0a75b8e3 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:08.474973007Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1213,"session_id":"b3dcf985-4eba-44cd-900b-fa9a3d49ab09"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:08.474982565Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1213,"client":"0","":"downstairs","session_id":"b3dcf985-4eba-44cd-900b-fa9a3d49ab09"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:08.474992294Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1213,"client":"1","":"downstairs","session_id":"b3dcf985-4eba-44cd-900b-fa9a3d49ab09"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:08.475002102Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1213,"client":"2","":"downstairs","session_id":"b3dcf985-4eba-44cd-900b-fa9a3d49ab09"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:08.475011901Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1213,"":"io task","client":"2","":"downstairs","session_id":"b3dcf985-4eba-44cd-900b-fa9a3d49ab09"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:08.475024159Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1213,"":"io task","client":"0","":"downstairs","session_id":"b3dcf985-4eba-44cd-900b-fa9a3d49ab09"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:08.475034397Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1213,"":"io task","client":"1","":"downstairs","session_id":"b3dcf985-4eba-44cd-900b-fa9a3d49ab09"} {"msg":"ds_connection connected from Ok(127.0.0.1:62252)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:08.475069052Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1213,"":"io task","client":"2","":"downstairs","session_id":"b3dcf985-4eba-44cd-900b-fa9a3d49ab09"} {"msg":"ds_connection connected from Ok(127.0.0.1:39698)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:08.475080191Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1213,"":"io task","client":"0","":"downstairs","session_id":"b3dcf985-4eba-44cd-900b-fa9a3d49ab09"} {"msg":"ds_connection connected from Ok(127.0.0.1:39785)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:08.475090509Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1213,"":"io task","client":"1","":"downstairs","session_id":"b3dcf985-4eba-44cd-900b-fa9a3d49ab09"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:08.475779767Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1213,"client":"1","":"downstairs","session_id":"b3dcf985-4eba-44cd-900b-fa9a3d49ab09"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:08.475801204Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1213,"client":"2","":"downstairs","session_id":"b3dcf985-4eba-44cd-900b-fa9a3d49ab09"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:08.475818142Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1213,"client":"0","":"downstairs","session_id":"b3dcf985-4eba-44cd-900b-fa9a3d49ab09"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:08.476091052Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1213,"session_id":"b3dcf985-4eba-44cd-900b-fa9a3d49ab09"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:08.476113918Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1213,"session_id":"b3dcf985-4eba-44cd-900b-fa9a3d49ab09"} {"msg":"[0]R flush_numbers[0..12]: [33, 30, 34, 35, 21, 21, 22, 38, 30, 31, 37, 34]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:08.476124447Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1213,"":"downstairs","":"downstairs","session_id":"b3dcf985-4eba-44cd-900b-fa9a3d49ab09"} {"msg":"[0]R generation[0..12]: [18, 16, 18, 18, 11, 11, 12, 19, 16, 17, 18, 18]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:08.476135415Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1213,"":"downstairs","":"downstairs","session_id":"b3dcf985-4eba-44cd-900b-fa9a3d49ab09"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:08.476146593Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1213,"":"downstairs","":"downstairs","session_id":"b3dcf985-4eba-44cd-900b-fa9a3d49ab09"} {"msg":"[1]R flush_numbers[0..12]: [33, 30, 34, 35, 21, 21, 22, 38, 30, 31, 37, 34]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:08.476156982Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1213,"":"downstairs","":"downstairs","session_id":"b3dcf985-4eba-44cd-900b-fa9a3d49ab09"} {"msg":"[1]R generation[0..12]: [18, 16, 18, 18, 11, 11, 12, 19, 16, 17, 18, 18]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:08.47617Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1213,"":"downstairs","":"downstairs","session_id":"b3dcf985-4eba-44cd-900b-fa9a3d49ab09"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:08.476181398Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1213,"":"downstairs","":"downstairs","session_id":"b3dcf985-4eba-44cd-900b-fa9a3d49ab09"} {"msg":"[2]R flush_numbers[0..12]: [33, 30, 34, 35, 21, 21, 22, 38, 30, 31, 37, 34]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:08.476191887Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1213,"":"downstairs","":"downstairs","session_id":"b3dcf985-4eba-44cd-900b-fa9a3d49ab09"} {"msg":"[2]R generation[0..12]: [18, 16, 18, 18, 11, 11, 12, 19, 16, 17, 18, 18]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:08.476202365Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1213,"":"downstairs","":"downstairs","session_id":"b3dcf985-4eba-44cd-900b-fa9a3d49ab09"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:08.476213543Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1213,"":"downstairs","":"downstairs","session_id":"b3dcf985-4eba-44cd-900b-fa9a3d49ab09"} {"msg":"Max found gen is 20","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:08.476223082Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1213,"":"downstairs","":"downstairs","session_id":"b3dcf985-4eba-44cd-900b-fa9a3d49ab09"} {"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:08.476232821Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1213,"":"downstairs","":"downstairs","session_id":"b3dcf985-4eba-44cd-900b-fa9a3d49ab09"} {"msg":"Next flush: 39","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:08.476242219Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1213,"":"downstairs","":"downstairs","session_id":"b3dcf985-4eba-44cd-900b-fa9a3d49ab09"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:08.476251708Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1213,"":"downstairs","":"downstairs","session_id":"b3dcf985-4eba-44cd-900b-fa9a3d49ab09"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:08.476261386Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1213,"session_id":"b3dcf985-4eba-44cd-900b-fa9a3d49ab09"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:08.476270165Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1213,"client":"0","":"downstairs","session_id":"b3dcf985-4eba-44cd-900b-fa9a3d49ab09"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:08.476280124Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1213,"client":"1","":"downstairs","session_id":"b3dcf985-4eba-44cd-900b-fa9a3d49ab09"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:08.476289792Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1213,"client":"2","":"downstairs","session_id":"b3dcf985-4eba-44cd-900b-fa9a3d49ab09"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:08.476299791Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1213,"session_id":"b3dcf985-4eba-44cd-900b-fa9a3d49ab09"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:08.476308989Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1213,"session_id":"b3dcf985-4eba-44cd-900b-fa9a3d49ab09"} {"msg":"11f48810-1f1c-4140-b10e-b0bb0a75b8e3 is now active with session: b3dcf985-4eba-44cd-900b-fa9a3d49ab09","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:08.476319108Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1213,"session_id":"b3dcf985-4eba-44cd-900b-fa9a3d49ab09"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:08.476329286Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1213,"session_id":"b3dcf985-4eba-44cd-900b-fa9a3d49ab09"} {"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:08.476338355Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1213} {"msg":"Activated sub_volume 11f48810-1f1c-4140-b10e-b0bb0a75b8e3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:08.476346714Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1213} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:08.476354993Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1213,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:08.47664487Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1213,"task":"crutest"} 01/30 Write block 524 len 40960 data: 2 2 2 2 2 2 2 2 2 2 02/30 Flush 03/30 Write block 134 len 20480 data: 3 2 2 2 3 04/30 Read block 338 len 32768 05/30 Write block 363 len 28672 data: 3 3 3 2 2 2 2 06/30 Read block 564 len 36864 07/30 Write block 286 len 4096 data: 5 08/30 Read block 8 len 40960 09/30 Write block 433 len 28672 data: 2 2 2 2 2 3 2 10/30 Write block 310 len 32768 data: 3 4 4 4 3 3 3 3 11/30 Read block 406 len 24576 12/30 Write block 314 len 12288 data: 4 4 4 13/30 Read block 577 len 28672 14/30 Read block 122 len 36864 15/30 Read block 397 len 40960 16/30 Read block 574 len 20480 17/30 Write block 431 len 36864 data: 2 2 3 3 3 3 3 4 3 18/30 Read block 307 len 20480 19/30 Read block 378 len 4096 20/30 Flush 21/30 Write block 95 len 4096 data: 2 22/30 Read block 54 len 4096 23/30 Read block 509 len 24576 24/30 Read block 363 len 36864 25/30 Write block 38 len 40960 data: 6 6 6 6 5 6 5 4 4 4 26/30 Write block 411 len 24576 data: 5 5 4 4 5 6 27/30 Read block 151 len 40960 28/30 Write block 476 len 4096 data: 2 29/30 Read block 51 len 12288 30/30 Read block 9 len 28672 ---------------------------------------------------------------- Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Sent Done Done false 1001 Acked Flush 0 Sent Done Done false 1002 Acked Write 5 Sent Done Done false 1003 Acked Read 8 Sent Done Done false 1004 Acked Write 7 Sent Done Done false 1005 Acked Read 9 Sent Done Done false 1006 Acked Write 1 Sent Done Done false 1007 Acked Read 10 Sent Done Done false 1008 Acked Write 7 Sent Done Done false 1009 Acked Write 8 Sent Done Done false 1010 Acked Read 6 Sent Done Done false 1011 Acked Write 3 Sent Done Done false 1012 Acked Read 7 Sent Done Done false 1013 Acked Read 9 Sent Done Done false 1014 Acked Read 10 Sent Done Done false 1015 Acked Read 5 Sent Done Done false 1016 Acked Write 9 Sent Done Done false 1017 Acked Read 5 Sent Done Done false 1018 Acked Read 1 Sent Done Done false 1019 Acked Flush 0 Sent Done Done false 1020 Acked Write 1 Sent Done Done false 1021 Acked Read 1 Sent Done Done false 1022 Acked Read 6 Sent Done Done false 1023 Acked Read 9 Sent Done Done false 1024 Acked Write 10 Sent Done Done false 1025 Acked Write 6 Sent Done Done false 1026 Acked Read 10 Sent Done Done false 1027 Acked Write 1 Sent Done Done false 1028 Acked Read 3 Sent Done Done false 1029 Acked Read 7 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 0 0 30 Done 0 30 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:54:08.486013228Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1213} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:08.486185672Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1213,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 18 18 18 33 33 33 F F F 1 020-039 16 16 16 30 30 30 F T T <--- 2 040-059 18 18 18 34 34 34 F T T <--- 3 060-079 18 18 18 35 35 35 F F F 4 080-099 11 11 11 21 21 21 F T T <--- 5 100-119 11 11 11 21 21 21 F F F 6 120-139 12 20 20 22 40 40 F F F <--- 7 140-159 19 19 19 38 38 38 F F F 8 160-179 16 16 16 30 30 30 F F F 9 180-199 17 17 17 31 31 31 F F F 10 200-219 18 18 18 37 37 37 F F F 11 220-239 18 18 18 34 34 34 F F F 12 240-259 17 17 17 31 31 31 F F F 13 260-279 18 18 18 34 34 34 F F F 14 280-299 16 20 20 30 40 40 F F F <--- 15 300-319 17 20 20 31 40 40 F F F <--- 16 320-339 14 14 14 28 28 28 F F F 17 340-359 19 19 19 38 38 38 F F F 18 360-379 12 20 20 22 40 40 F F F <--- 19 380-399 19 19 19 38 38 38 F F F 20 400-419 16 16 16 30 30 30 F T T <--- 21 420-439 16 20 20 30 40 40 F F F <--- 22 440-459 19 19 19 38 38 38 F F F 23 460-479 13 13 13 23 23 23 F T T <--- 24 480-499 18 18 18 36 36 36 F F F 25 500-519 17 17 17 31 31 31 F F F 26 520-539 3 20 20 3 39 39 F F F <--- 27 540-559 18 18 18 35 35 35 F F F 28 560-579 16 16 16 30 30 30 F F F 29 580-599 19 19 19 38 38 38 F F F Max gen: 20, Max flush: 40 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 10 Verifying data now No change after verify Loop: 10 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 18 18 18 33 33 33 F F F 1 020-039 21 21 21 41 41 41 F F F 2 040-059 21 21 21 41 41 41 F F F 3 060-079 18 18 18 35 35 35 F F F 4 080-099 21 21 21 41 41 41 F F F 5 100-119 11 11 11 21 21 21 F F F 6 120-139 20 20 20 40 40 40 F F F 7 140-159 19 19 19 38 38 38 F F F 8 160-179 16 16 16 30 30 30 F F F 9 180-199 17 17 17 31 31 31 F F F 10 200-219 18 18 18 37 37 37 F F F 11 220-239 18 18 18 34 34 34 F F F 12 240-259 17 17 17 31 31 31 F F F 13 260-279 18 18 18 34 34 34 F F F 14 280-299 20 20 20 40 40 40 F F F 15 300-319 20 20 20 40 40 40 F F F 16 320-339 14 14 14 28 28 28 F F F 17 340-359 19 19 19 38 38 38 F F F 18 360-379 20 20 20 40 40 40 F F F 19 380-399 19 19 19 38 38 38 F F F 20 400-419 21 21 21 41 41 41 F F F 21 420-439 20 20 20 40 40 40 F F F 22 440-459 19 19 19 38 38 38 F F F 23 460-479 21 21 21 41 41 41 F F F 24 480-499 18 18 18 36 36 36 F F F 25 500-519 17 17 17 31 31 31 F F F 26 520-539 20 20 20 39 39 39 F F F 27 540-559 18 18 18 35 35 35 F F F 28 560-579 16 16 16 30 30 30 F F F 29 580-599 19 19 19 38 38 38 F F F Max gen: 21, Max flush: 41 Begin loop 11 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:12.735379145Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1221,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:12.743859884Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1221,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:12.744160189Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1221} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:12.744186295Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1221} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:12.744194864Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1221} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:12.744206293Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1221} {"msg":"Crucible 0343da43-8e41-4e32-9b7d-8c65850b3941 has session id: fd77973f-4f18-4cef-8ad5-843c31e5b94f","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:12.744586216Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1221,"session_id":"fd77973f-4f18-4cef-8ad5-843c31e5b94f"} {"msg":"Upstairs opts: Upstairs UUID: 0343da43-8e41-4e32-9b7d-8c65850b3941, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:12.744615932Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1221,"session_id":"fd77973f-4f18-4cef-8ad5-843c31e5b94f"} {"msg":"Crucible stats registered with UUID: 0343da43-8e41-4e32-9b7d-8c65850b3941","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:12.74462706Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1221,"session_id":"fd77973f-4f18-4cef-8ad5-843c31e5b94f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:54:12.744636479Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1221,"session_id":"fd77973f-4f18-4cef-8ad5-843c31e5b94f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:12.744775898Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1221,"":"io task","client":"1","":"downstairs","session_id":"fd77973f-4f18-4cef-8ad5-843c31e5b94f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:12.744805374Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1221,"":"io task","client":"0","":"downstairs","session_id":"fd77973f-4f18-4cef-8ad5-843c31e5b94f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:12.744816263Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1221,"":"io task","client":"2","":"downstairs","session_id":"fd77973f-4f18-4cef-8ad5-843c31e5b94f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:12.744927976Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1221,"task":"crutest"} {"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:12.744942934Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1221} {"msg":"0343da43-8e41-4e32-9b7d-8c65850b3941 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:12.744983998Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1221,"session_id":"fd77973f-4f18-4cef-8ad5-843c31e5b94f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:12.744993346Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1221,"client":"0","":"downstairs","session_id":"fd77973f-4f18-4cef-8ad5-843c31e5b94f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:12.745003215Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1221,"client":"1","":"downstairs","session_id":"fd77973f-4f18-4cef-8ad5-843c31e5b94f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:12.745012884Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1221,"client":"2","":"downstairs","session_id":"fd77973f-4f18-4cef-8ad5-843c31e5b94f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:12.745022642Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1221,"":"io task","client":"2","":"downstairs","session_id":"fd77973f-4f18-4cef-8ad5-843c31e5b94f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:12.745032711Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1221,"":"io task","client":"1","":"downstairs","session_id":"fd77973f-4f18-4cef-8ad5-843c31e5b94f"} {"msg":"ds_connection connected from Ok(127.0.0.1:45056)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:12.745109079Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1221,"":"io task","client":"2","":"downstairs","session_id":"fd77973f-4f18-4cef-8ad5-843c31e5b94f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:12.745119538Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1221,"":"io task","client":"0","":"downstairs","session_id":"fd77973f-4f18-4cef-8ad5-843c31e5b94f"} {"msg":"ds_connection connected from Ok(127.0.0.1:50693)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:12.745129896Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1221,"":"io task","client":"1","":"downstairs","session_id":"fd77973f-4f18-4cef-8ad5-843c31e5b94f"} {"msg":"ds_connection connected from Ok(127.0.0.1:34755)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:12.745140665Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1221,"":"io task","client":"0","":"downstairs","session_id":"fd77973f-4f18-4cef-8ad5-843c31e5b94f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:12.745666537Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1221,"client":"2","":"downstairs","session_id":"fd77973f-4f18-4cef-8ad5-843c31e5b94f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:12.745755414Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1221,"client":"0","":"downstairs","session_id":"fd77973f-4f18-4cef-8ad5-843c31e5b94f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:12.745768562Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1221,"client":"1","":"downstairs","session_id":"fd77973f-4f18-4cef-8ad5-843c31e5b94f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:12.745942716Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1221,"session_id":"fd77973f-4f18-4cef-8ad5-843c31e5b94f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:12.74605107Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1221,"session_id":"fd77973f-4f18-4cef-8ad5-843c31e5b94f"} {"msg":"[0]R flush_numbers[0..12]: [33, 41, 41, 35, 41, 21, 40, 38, 30, 31, 37, 34]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:12.74618792Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1221,"":"downstairs","":"downstairs","session_id":"fd77973f-4f18-4cef-8ad5-843c31e5b94f"} {"msg":"[0]R generation[0..12]: [18, 21, 21, 18, 21, 11, 20, 19, 16, 17, 18, 18]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:12.746199598Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1221,"":"downstairs","":"downstairs","session_id":"fd77973f-4f18-4cef-8ad5-843c31e5b94f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:12.746210867Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1221,"":"downstairs","":"downstairs","session_id":"fd77973f-4f18-4cef-8ad5-843c31e5b94f"} {"msg":"[1]R flush_numbers[0..12]: [33, 41, 41, 35, 41, 21, 40, 38, 30, 31, 37, 34]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:12.746221165Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1221,"":"downstairs","":"downstairs","session_id":"fd77973f-4f18-4cef-8ad5-843c31e5b94f"} {"msg":"[1]R generation[0..12]: [18, 21, 21, 18, 21, 11, 20, 19, 16, 17, 18, 18]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:12.746231324Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1221,"":"downstairs","":"downstairs","session_id":"fd77973f-4f18-4cef-8ad5-843c31e5b94f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:12.746242322Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1221,"":"downstairs","":"downstairs","session_id":"fd77973f-4f18-4cef-8ad5-843c31e5b94f"} {"msg":"[2]R flush_numbers[0..12]: [33, 41, 41, 35, 41, 21, 40, 38, 30, 31, 37, 34]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:12.746252841Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1221,"":"downstairs","":"downstairs","session_id":"fd77973f-4f18-4cef-8ad5-843c31e5b94f"} {"msg":"[2]R generation[0..12]: [18, 21, 21, 18, 21, 11, 20, 19, 16, 17, 18, 18]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:12.746263129Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1221,"":"downstairs","":"downstairs","session_id":"fd77973f-4f18-4cef-8ad5-843c31e5b94f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:12.746273857Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1221,"":"downstairs","":"downstairs","session_id":"fd77973f-4f18-4cef-8ad5-843c31e5b94f"} {"msg":"Max found gen is 22","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:12.746283466Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1221,"":"downstairs","":"downstairs","session_id":"fd77973f-4f18-4cef-8ad5-843c31e5b94f"} {"msg":"Generation requested: 22 >= found:22","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:12.746296194Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1221,"":"downstairs","":"downstairs","session_id":"fd77973f-4f18-4cef-8ad5-843c31e5b94f"} {"msg":"Next flush: 42","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:12.746305673Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1221,"":"downstairs","":"downstairs","session_id":"fd77973f-4f18-4cef-8ad5-843c31e5b94f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:12.746315031Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1221,"":"downstairs","":"downstairs","session_id":"fd77973f-4f18-4cef-8ad5-843c31e5b94f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:12.74632486Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1221,"session_id":"fd77973f-4f18-4cef-8ad5-843c31e5b94f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:12.746333599Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1221,"client":"0","":"downstairs","session_id":"fd77973f-4f18-4cef-8ad5-843c31e5b94f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:12.746343617Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1221,"client":"1","":"downstairs","session_id":"fd77973f-4f18-4cef-8ad5-843c31e5b94f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:12.746353586Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1221,"client":"2","":"downstairs","session_id":"fd77973f-4f18-4cef-8ad5-843c31e5b94f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:12.746363774Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1221,"session_id":"fd77973f-4f18-4cef-8ad5-843c31e5b94f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:12.746373033Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1221,"session_id":"fd77973f-4f18-4cef-8ad5-843c31e5b94f"} {"msg":"0343da43-8e41-4e32-9b7d-8c65850b3941 is now active with session: fd77973f-4f18-4cef-8ad5-843c31e5b94f","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:12.746382881Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1221,"session_id":"fd77973f-4f18-4cef-8ad5-843c31e5b94f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:12.74639161Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1221,"session_id":"fd77973f-4f18-4cef-8ad5-843c31e5b94f"} {"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:12.746400639Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1221} {"msg":"Activated sub_volume 0343da43-8e41-4e32-9b7d-8c65850b3941","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:12.746408618Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1221} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:12.746416866Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1221,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:12.74673153Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1221,"task":"crutest"} 01/30 Flush 02/30 Write block 499 len 28672 data: 2 2 2 2 2 2 2 03/30 Read block 331 len 4096 04/30 Read block 166 len 36864 05/30 Read block 247 len 16384 06/30 Flush 07/30 Write block 502 len 12288 data: 3 3 3 08/30 Read block 347 len 12288 09/30 Flush 10/30 Write block 88 len 4096 data: 5 11/30 Write block 189 len 24576 data: 2 2 3 3 3 3 12/30 Read block 121 len 36864 13/30 Read block 104 len 40960 14/30 Write block 90 len 36864 data: 5 4 3 3 2 3 3 3 4 15/30 Write block 195 len 12288 data: 2 2 3 16/30 Read block 447 len 8192 17/30 Write block 406 len 32768 data: 2 2 2 2 5 6 6 5 18/30 Read block 157 len 36864 19/30 Write block 25 len 12288 data: 4 3 3 20/30 Read block 393 len 32768 21/30 Read block 397 len 24576 22/30 Read block 41 len 24576 23/30 Write block 24 len 36864 data: 4 5 4 4 4 4 3 3 3 24/30 Write block 488 len 12288 data: 5 3 3 25/30 Read block 539 len 8192 26/30 Flush 27/30 Write block 134 len 36864 data: 4 3 3 3 4 3 3 4 3 28/30 Write block 518 len 24576 data: 5 4 3 3 3 2 29/30 Read block 470 len 28672 30/30 Read block 255 len 36864 ---------------------------------------------------------------- Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 7 Done Sent Done false 1002 Acked Read 1 Done Sent Done false 1003 Acked Read 9 Done Sent Done false 1004 Acked Read 4 Done Sent Done false 1005 Acked Flush 0 Done Sent Done false 1006 Acked Write 3 Done Sent Done false 1007 Acked Read 3 Done Sent Done false 1008 Acked Flush 0 Done Sent Done false 1009 Acked Write 1 Done Sent Done false 1010 Acked Write 6 Done Sent Done false 1011 Acked Read 9 Done Sent Done false 1012 Acked Read 10 Done Sent Done false 1013 Acked Write 9 Done Sent Done false 1014 Acked Write 3 Done Sent Done false 1015 Acked Read 2 Done Sent Done false 1016 Acked Write 8 Done Sent Done false 1017 Acked Read 9 Done Sent Done false 1018 Acked Write 3 Done Sent Done false 1019 Acked Read 8 Done Sent Done false 1020 Acked Read 6 Done Sent Done false 1021 Acked Read 6 Done Sent Done false 1022 Acked Write 9 Done Sent Done false 1023 Acked Write 3 Done Sent Done false 1024 Acked Read 2 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 9 Done Sent Done false 1027 Acked Write 6 Done Sent Done false 1028 Acked Read 7 Done Sent Done false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 29 1 30 Done 29 0 28 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1000)) Some(JobId(1025)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:54:12.756343832Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1221} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:12.756520436Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1221,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 18 18 18 33 33 33 F F F 1 020-039 22 21 22 45 41 45 F F F <--- 2 040-059 21 21 21 41 41 41 F F F 3 060-079 18 18 18 35 35 35 F F F 4 080-099 22 21 22 45 41 45 F F F <--- 5 100-119 11 11 11 21 21 21 F F F 6 120-139 20 20 20 40 40 40 T F T <--- 7 140-159 19 19 19 38 38 38 T F T <--- 8 160-179 16 16 16 30 30 30 F F F 9 180-199 22 17 22 45 31 45 F F F <--- 10 200-219 18 18 18 37 37 37 F F F 11 220-239 18 18 18 34 34 34 F F F 12 240-259 17 17 17 31 31 31 F F F 13 260-279 18 18 18 34 34 34 F F F 14 280-299 20 20 20 40 40 40 F F F 15 300-319 20 20 20 40 40 40 F F F 16 320-339 14 14 14 28 28 28 F F F 17 340-359 19 19 19 38 38 38 F F F 18 360-379 20 20 20 40 40 40 F F F 19 380-399 19 19 19 38 38 38 F F F 20 400-419 22 21 22 45 41 45 F F F <--- 21 420-439 20 20 20 40 40 40 F F F 22 440-459 19 19 19 38 38 38 F F F 23 460-479 21 21 21 41 41 41 F F F 24 480-499 22 18 22 45 36 45 F F F <--- 25 500-519 22 17 22 44 31 44 T F T <--- 26 520-539 20 20 20 39 39 39 T F T <--- 27 540-559 18 18 18 35 35 35 F F F 28 560-579 16 16 16 30 30 30 F F F 29 580-599 19 19 19 38 38 38 F F F Max gen: 22, Max flush: 45 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 11 Verifying data now No change after verify Loop: 11 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 18 18 18 33 33 33 F F F 1 020-039 22 22 22 45 45 45 F F F 2 040-059 21 21 21 41 41 41 F F F 3 060-079 18 18 18 35 35 35 F F F 4 080-099 22 22 22 45 45 45 F F F 5 100-119 11 11 11 21 21 21 F F F 6 120-139 23 23 23 46 46 46 F F F 7 140-159 23 23 23 46 46 46 F F F 8 160-179 16 16 16 30 30 30 F F F 9 180-199 22 22 22 45 45 45 F F F 10 200-219 18 18 18 37 37 37 F F F 11 220-239 18 18 18 34 34 34 F F F 12 240-259 17 17 17 31 31 31 F F F 13 260-279 18 18 18 34 34 34 F F F 14 280-299 20 20 20 40 40 40 F F F 15 300-319 20 20 20 40 40 40 F F F 16 320-339 14 14 14 28 28 28 F F F 17 340-359 19 19 19 38 38 38 F F F 18 360-379 20 20 20 40 40 40 F F F 19 380-399 19 19 19 38 38 38 F F F 20 400-419 22 22 22 45 45 45 F F F 21 420-439 20 20 20 40 40 40 F F F 22 440-459 19 19 19 38 38 38 F F F 23 460-479 21 21 21 41 41 41 F F F 24 480-499 22 22 22 45 45 45 F F F 25 500-519 23 23 23 46 46 46 F F F 26 520-539 23 23 23 46 46 46 F F F 27 540-559 18 18 18 35 35 35 F F F 28 560-579 16 16 16 30 30 30 F F F 29 580-599 19 19 19 38 38 38 F F F Max gen: 23, Max flush: 46 Begin loop 12 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:16.995999751Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1229,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:17.004053643Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1229,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:17.004362587Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1229} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:17.004386394Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1229} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:17.004398092Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1229} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:17.00440984Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1229} {"msg":"Crucible 60767f11-0227-4d2a-9433-3d54bc52c67d has session id: 0b297c2d-6c74-403a-935c-13e38c90f72b","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:17.004768107Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1229,"session_id":"0b297c2d-6c74-403a-935c-13e38c90f72b"} {"msg":"Upstairs opts: Upstairs UUID: 60767f11-0227-4d2a-9433-3d54bc52c67d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:17.004796623Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1229,"session_id":"0b297c2d-6c74-403a-935c-13e38c90f72b"} {"msg":"Crucible stats registered with UUID: 60767f11-0227-4d2a-9433-3d54bc52c67d","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:17.004807891Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1229,"session_id":"0b297c2d-6c74-403a-935c-13e38c90f72b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:54:17.00481719Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1229,"session_id":"0b297c2d-6c74-403a-935c-13e38c90f72b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:17.004962199Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1229,"":"io task","client":"1","":"downstairs","session_id":"0b297c2d-6c74-403a-935c-13e38c90f72b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:17.004991644Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1229,"":"io task","client":"0","":"downstairs","session_id":"0b297c2d-6c74-403a-935c-13e38c90f72b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:17.005002763Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1229,"":"io task","client":"2","":"downstairs","session_id":"0b297c2d-6c74-403a-935c-13e38c90f72b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:17.005143722Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1229,"task":"crutest"} {"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:17.00515344Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1229} {"msg":"60767f11-0227-4d2a-9433-3d54bc52c67d active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:17.005170838Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1229,"session_id":"0b297c2d-6c74-403a-935c-13e38c90f72b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:17.005179936Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1229,"client":"0","":"downstairs","session_id":"0b297c2d-6c74-403a-935c-13e38c90f72b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:17.005191625Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1229,"client":"1","":"downstairs","session_id":"0b297c2d-6c74-403a-935c-13e38c90f72b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:17.005201503Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1229,"client":"2","":"downstairs","session_id":"0b297c2d-6c74-403a-935c-13e38c90f72b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:17.005211292Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1229,"":"io task","client":"2","":"downstairs","session_id":"0b297c2d-6c74-403a-935c-13e38c90f72b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:17.00522131Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1229,"":"io task","client":"0","":"downstairs","session_id":"0b297c2d-6c74-403a-935c-13e38c90f72b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:17.005231869Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1229,"":"io task","client":"1","":"downstairs","session_id":"0b297c2d-6c74-403a-935c-13e38c90f72b"} {"msg":"ds_connection connected from Ok(127.0.0.1:60757)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:17.005249956Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1229,"":"io task","client":"2","":"downstairs","session_id":"0b297c2d-6c74-403a-935c-13e38c90f72b"} {"msg":"ds_connection connected from Ok(127.0.0.1:55565)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:17.005275722Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1229,"":"io task","client":"0","":"downstairs","session_id":"0b297c2d-6c74-403a-935c-13e38c90f72b"} {"msg":"ds_connection connected from Ok(127.0.0.1:38820)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:17.0052946Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1229,"":"io task","client":"1","":"downstairs","session_id":"0b297c2d-6c74-403a-935c-13e38c90f72b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:17.006020372Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1229,"client":"1","":"downstairs","session_id":"0b297c2d-6c74-403a-935c-13e38c90f72b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:17.006047009Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1229,"client":"0","":"downstairs","session_id":"0b297c2d-6c74-403a-935c-13e38c90f72b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:17.006058967Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1229,"client":"2","":"downstairs","session_id":"0b297c2d-6c74-403a-935c-13e38c90f72b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:17.006318758Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1229,"session_id":"0b297c2d-6c74-403a-935c-13e38c90f72b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:17.006354793Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1229,"session_id":"0b297c2d-6c74-403a-935c-13e38c90f72b"} {"msg":"[0]R flush_numbers[0..12]: [33, 45, 41, 35, 45, 21, 46, 46, 30, 45, 37, 34]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:17.006408935Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1229,"":"downstairs","":"downstairs","session_id":"0b297c2d-6c74-403a-935c-13e38c90f72b"} {"msg":"[0]R generation[0..12]: [18, 22, 21, 18, 22, 11, 23, 23, 16, 22, 18, 18]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:17.006420203Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1229,"":"downstairs","":"downstairs","session_id":"0b297c2d-6c74-403a-935c-13e38c90f72b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:17.006433961Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1229,"":"downstairs","":"downstairs","session_id":"0b297c2d-6c74-403a-935c-13e38c90f72b"} {"msg":"[1]R flush_numbers[0..12]: [33, 45, 41, 35, 45, 21, 46, 46, 30, 45, 37, 34]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:17.00644434Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1229,"":"downstairs","":"downstairs","session_id":"0b297c2d-6c74-403a-935c-13e38c90f72b"} {"msg":"[1]R generation[0..12]: [18, 22, 21, 18, 22, 11, 23, 23, 16, 22, 18, 18]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:17.006455018Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1229,"":"downstairs","":"downstairs","session_id":"0b297c2d-6c74-403a-935c-13e38c90f72b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:17.006466257Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1229,"":"downstairs","":"downstairs","session_id":"0b297c2d-6c74-403a-935c-13e38c90f72b"} {"msg":"[2]R flush_numbers[0..12]: [33, 45, 41, 35, 45, 21, 46, 46, 30, 45, 37, 34]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:17.006476785Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1229,"":"downstairs","":"downstairs","session_id":"0b297c2d-6c74-403a-935c-13e38c90f72b"} {"msg":"[2]R generation[0..12]: [18, 22, 21, 18, 22, 11, 23, 23, 16, 22, 18, 18]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:17.006487134Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1229,"":"downstairs","":"downstairs","session_id":"0b297c2d-6c74-403a-935c-13e38c90f72b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:17.006498142Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1229,"":"downstairs","":"downstairs","session_id":"0b297c2d-6c74-403a-935c-13e38c90f72b"} {"msg":"Max found gen is 24","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:17.006507781Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1229,"":"downstairs","":"downstairs","session_id":"0b297c2d-6c74-403a-935c-13e38c90f72b"} {"msg":"Generation requested: 24 >= found:24","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:17.006517609Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1229,"":"downstairs","":"downstairs","session_id":"0b297c2d-6c74-403a-935c-13e38c90f72b"} {"msg":"Next flush: 47","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:17.006526838Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1229,"":"downstairs","":"downstairs","session_id":"0b297c2d-6c74-403a-935c-13e38c90f72b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:17.006536616Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1229,"":"downstairs","":"downstairs","session_id":"0b297c2d-6c74-403a-935c-13e38c90f72b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:17.006546545Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1229,"session_id":"0b297c2d-6c74-403a-935c-13e38c90f72b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:17.006555374Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1229,"client":"0","":"downstairs","session_id":"0b297c2d-6c74-403a-935c-13e38c90f72b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:17.006565032Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1229,"client":"1","":"downstairs","session_id":"0b297c2d-6c74-403a-935c-13e38c90f72b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:17.006574711Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1229,"client":"2","":"downstairs","session_id":"0b297c2d-6c74-403a-935c-13e38c90f72b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:17.006585609Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1229,"session_id":"0b297c2d-6c74-403a-935c-13e38c90f72b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:17.006594778Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1229,"session_id":"0b297c2d-6c74-403a-935c-13e38c90f72b"} {"msg":"60767f11-0227-4d2a-9433-3d54bc52c67d is now active with session: 0b297c2d-6c74-403a-935c-13e38c90f72b","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:17.006604766Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1229,"session_id":"0b297c2d-6c74-403a-935c-13e38c90f72b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:17.006613585Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1229,"session_id":"0b297c2d-6c74-403a-935c-13e38c90f72b"} {"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:17.006622714Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1229} {"msg":"Activated sub_volume 60767f11-0227-4d2a-9433-3d54bc52c67d","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:17.006631112Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1229} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:17.006639371Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1229,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:17.006908301Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1229,"task":"crutest"} 01/30 Write block 390 len 24576 data: 5 5 5 4 4 4 02/30 Flush 03/30 Write block 176 len 12288 data: 5 4 4 04/30 Write block 591 len 24576 data: 4 5 6 5 4 3 05/30 Write block 432 len 16384 data: 3 4 4 4 06/30 Read block 248 len 24576 07/30 Flush 08/30 Write block 212 len 12288 data: 3 3 3 09/30 Write block 311 len 24576 data: 5 5 5 5 5 5 10/30 Write block 309 len 28672 data: 3 4 6 6 6 6 6 11/30 Flush 12/30 Write block 583 len 20480 data: 4 5 5 4 2 13/30 Write block 134 len 12288 data: 5 4 4 14/30 Write block 427 len 24576 data: 2 2 2 2 3 4 15/30 Read block 284 len 20480 16/30 Read block 107 len 20480 17/30 Write block 395 len 12288 data: 5 3 3 18/30 Read block 49 len 36864 19/30 Read block 141 len 36864 20/30 Read block 537 len 8192 21/30 Read block 278 len 40960 22/30 Read block 185 len 24576 23/30 Write block 191 len 28672 data: 4 4 4 4 3 3 4 24/30 Write block 135 len 24576 data: 5 5 4 5 4 4 25/30 Write block 100 len 8192 data: 4 4 26/30 Flush 27/30 Write block 422 len 32768 data: 3 3 3 3 2 3 3 3 28/30 Read block 225 len 36864 29/30 Write block 155 len 20480 data: 3 4 5 4 3 30/30 Read block 90 len 28672 ---------------------------------------------------------------- Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 3 Done Done Done false 1003 Acked Write 6 Done Done Sent false 1004 Acked Write 4 Done Done Sent false 1005 Acked Read 6 Done Done Sent false 1006 Acked Flush 0 Done Done Sent false 1007 Acked Write 3 Done Done Sent false 1008 Acked Write 6 Done Done Sent false 1009 Acked Write 7 Done Done Sent false 1010 Acked Flush 0 Done Done Sent false 1011 Acked Write 5 Done Done Sent false 1012 Acked Write 3 Done Done Sent false 1013 Acked Write 6 Done Done Sent false 1014 Acked Read 5 Done Done Sent false 1015 Acked Read 5 Done Done Sent false 1016 Acked Write 3 Done Done Sent false 1017 Acked Read 9 Done Done Sent false 1018 Acked Read 9 Done Done Sent false 1019 Acked Read 2 Done Done Sent false 1020 Acked Read 10 Done Done Sent false 1021 Acked Read 6 Done Done Sent false 1022 Acked Write 7 Done Done Sent false 1023 Acked Write 6 Done Done Sent false 1024 Acked Write 2 Done Done Sent false 1025 Acked Flush 0 Done Done Sent false 1026 Acked Write 8 Done Done Sent false 1027 Acked Read 9 Done Done Sent false 1028 Acked Write 5 Done Done Sent false 1029 Acked Read 7 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 27 27 Done 28 28 1 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1001)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:54:17.015476357Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1229} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:17.015644853Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1229,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 18 18 18 33 33 33 F F F 1 020-039 22 22 22 45 45 45 F F F 2 040-059 21 21 21 41 41 41 F F F 3 060-079 18 18 18 35 35 35 F F F 4 080-099 22 22 22 45 45 45 F F F 5 100-119 24 24 11 50 50 21 F F F <--- 6 120-139 24 24 23 50 50 46 F F F <--- 7 140-159 24 24 23 50 50 46 T T F <--- 8 160-179 24 24 16 48 48 30 F F T <--- 9 180-199 24 24 22 50 50 45 F F F <--- 10 200-219 24 24 18 49 49 37 F F F <--- 11 220-239 18 18 18 34 34 34 F F F 12 240-259 17 17 17 31 31 31 F F F 13 260-279 18 18 18 34 34 34 F F F 14 280-299 20 20 20 40 40 40 F F F 15 300-319 24 24 20 49 49 40 F F F <--- 16 320-339 14 14 14 28 28 28 F F F 17 340-359 19 19 19 38 38 38 F F F 18 360-379 20 20 20 40 40 40 F F F 19 380-399 24 24 24 50 50 47 F F F <--- 20 400-419 22 22 22 45 45 45 F F F 21 420-439 24 24 20 50 50 40 T T F <--- 22 440-459 19 19 19 38 38 38 F F F 23 460-479 21 21 21 41 41 41 F F F 24 480-499 22 22 22 45 45 45 F F F 25 500-519 23 23 23 46 46 46 F F F 26 520-539 23 23 23 46 46 46 F F F 27 540-559 18 18 18 35 35 35 F F F 28 560-579 16 16 16 30 30 30 F F F 29 580-599 24 24 19 50 50 38 F F F <--- Max gen: 24, Max flush: 50 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 12 Verifying data now No change after verify Loop: 12 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 18 18 18 33 33 33 F F F 1 020-039 22 22 22 45 45 45 F F F 2 040-059 21 21 21 41 41 41 F F F 3 060-079 18 18 18 35 35 35 F F F 4 080-099 22 22 22 45 45 45 F F F 5 100-119 24 24 24 50 50 50 F F F 6 120-139 24 24 24 50 50 50 F F F 7 140-159 25 25 25 51 51 51 F F F 8 160-179 24 24 24 48 48 48 F F F 9 180-199 24 24 24 50 50 50 F F F 10 200-219 24 24 24 49 49 49 F F F 11 220-239 18 18 18 34 34 34 F F F 12 240-259 17 17 17 31 31 31 F F F 13 260-279 18 18 18 34 34 34 F F F 14 280-299 20 20 20 40 40 40 F F F 15 300-319 24 24 24 49 49 49 F F F 16 320-339 14 14 14 28 28 28 F F F 17 340-359 19 19 19 38 38 38 F F F 18 360-379 20 20 20 40 40 40 F F F 19 380-399 24 24 24 50 50 50 F F F 20 400-419 22 22 22 45 45 45 F F F 21 420-439 25 25 25 51 51 51 F F F 22 440-459 19 19 19 38 38 38 F F F 23 460-479 21 21 21 41 41 41 F F F 24 480-499 22 22 22 45 45 45 F F F 25 500-519 23 23 23 46 46 46 F F F 26 520-539 23 23 23 46 46 46 F F F 27 540-559 18 18 18 35 35 35 F F F 28 560-579 16 16 16 30 30 30 F F F 29 580-599 24 24 24 50 50 50 F F F Max gen: 25, Max flush: 51 Begin loop 13 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:21.255481906Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1237,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:54:21.262584518Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1237,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:21.262640299Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1237,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:21.269538682Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1237,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:21.26982434Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1237} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:21.269849676Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1237} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:21.269858195Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1237} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:21.269869633Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1237} {"msg":"Crucible 520e6f7f-3caf-4963-82c5-cbe2b8852a4b has session id: 44450e7d-56fd-4fbc-8233-ff59000df1fb","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:21.27022684Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1237,"session_id":"44450e7d-56fd-4fbc-8233-ff59000df1fb"} {"msg":"Upstairs opts: Upstairs UUID: 520e6f7f-3caf-4963-82c5-cbe2b8852a4b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:21.270260885Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1237,"session_id":"44450e7d-56fd-4fbc-8233-ff59000df1fb"} {"msg":"Crucible stats registered with UUID: 520e6f7f-3caf-4963-82c5-cbe2b8852a4b","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:21.270272553Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1237,"session_id":"44450e7d-56fd-4fbc-8233-ff59000df1fb"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:54:21.270281912Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1237,"session_id":"44450e7d-56fd-4fbc-8233-ff59000df1fb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:21.270393936Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1237,"":"io task","client":"0","":"downstairs","session_id":"44450e7d-56fd-4fbc-8233-ff59000df1fb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:21.270410303Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1237,"":"io task","client":"1","":"downstairs","session_id":"44450e7d-56fd-4fbc-8233-ff59000df1fb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:21.270420312Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1237,"":"io task","client":"2","":"downstairs","session_id":"44450e7d-56fd-4fbc-8233-ff59000df1fb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:21.270575639Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1237,"task":"crutest"} {"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:21.270594006Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1237} {"msg":"520e6f7f-3caf-4963-82c5-cbe2b8852a4b active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:21.270617593Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1237,"session_id":"44450e7d-56fd-4fbc-8233-ff59000df1fb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:21.270626931Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1237,"client":"0","":"downstairs","session_id":"44450e7d-56fd-4fbc-8233-ff59000df1fb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:21.27063678Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1237,"client":"1","":"downstairs","session_id":"44450e7d-56fd-4fbc-8233-ff59000df1fb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:21.270646438Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1237,"client":"2","":"downstairs","session_id":"44450e7d-56fd-4fbc-8233-ff59000df1fb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:21.270656207Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1237,"":"io task","client":"2","":"downstairs","session_id":"44450e7d-56fd-4fbc-8233-ff59000df1fb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:21.270666415Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1237,"":"io task","client":"0","":"downstairs","session_id":"44450e7d-56fd-4fbc-8233-ff59000df1fb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:21.27070081Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1237,"":"io task","client":"1","":"downstairs","session_id":"44450e7d-56fd-4fbc-8233-ff59000df1fb"} {"msg":"ds_connection connected from Ok(127.0.0.1:48073)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:21.270727386Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1237,"":"io task","client":"2","":"downstairs","session_id":"44450e7d-56fd-4fbc-8233-ff59000df1fb"} {"msg":"ds_connection connected from Ok(127.0.0.1:58446)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:21.270738895Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1237,"":"io task","client":"1","":"downstairs","session_id":"44450e7d-56fd-4fbc-8233-ff59000df1fb"} {"msg":"ds_connection connected from Ok(127.0.0.1:53691)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:21.270749273Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1237,"":"io task","client":"0","":"downstairs","session_id":"44450e7d-56fd-4fbc-8233-ff59000df1fb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:21.271463488Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1237,"client":"2","":"downstairs","session_id":"44450e7d-56fd-4fbc-8233-ff59000df1fb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:21.271484115Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1237,"client":"1","":"downstairs","session_id":"44450e7d-56fd-4fbc-8233-ff59000df1fb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:21.271580201Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1237,"client":"0","":"downstairs","session_id":"44450e7d-56fd-4fbc-8233-ff59000df1fb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:21.271832933Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1237,"session_id":"44450e7d-56fd-4fbc-8233-ff59000df1fb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:21.271883436Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1237,"session_id":"44450e7d-56fd-4fbc-8233-ff59000df1fb"} {"msg":"[0]R flush_numbers[0..12]: [33, 45, 41, 35, 45, 50, 50, 51, 48, 50, 49, 34]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:21.271927209Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1237,"":"downstairs","":"downstairs","session_id":"44450e7d-56fd-4fbc-8233-ff59000df1fb"} {"msg":"[0]R generation[0..12]: [18, 22, 21, 18, 22, 24, 24, 25, 24, 24, 24, 18]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:21.271938458Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1237,"":"downstairs","":"downstairs","session_id":"44450e7d-56fd-4fbc-8233-ff59000df1fb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:21.271949646Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1237,"":"downstairs","":"downstairs","session_id":"44450e7d-56fd-4fbc-8233-ff59000df1fb"} {"msg":"[1]R flush_numbers[0..12]: [33, 45, 41, 35, 45, 50, 50, 51, 48, 50, 49, 34]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:21.271960004Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1237,"":"downstairs","":"downstairs","session_id":"44450e7d-56fd-4fbc-8233-ff59000df1fb"} {"msg":"[1]R generation[0..12]: [18, 22, 21, 18, 22, 24, 24, 25, 24, 24, 24, 18]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:21.271970353Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1237,"":"downstairs","":"downstairs","session_id":"44450e7d-56fd-4fbc-8233-ff59000df1fb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:21.271981251Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1237,"":"downstairs","":"downstairs","session_id":"44450e7d-56fd-4fbc-8233-ff59000df1fb"} {"msg":"[2]R flush_numbers[0..12]: [33, 45, 41, 35, 45, 50, 50, 51, 48, 50, 49, 34]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:21.27199356Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1237,"":"downstairs","":"downstairs","session_id":"44450e7d-56fd-4fbc-8233-ff59000df1fb"} {"msg":"[2]R generation[0..12]: [18, 22, 21, 18, 22, 24, 24, 25, 24, 24, 24, 18]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:21.272004298Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1237,"":"downstairs","":"downstairs","session_id":"44450e7d-56fd-4fbc-8233-ff59000df1fb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:21.272015296Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1237,"":"downstairs","":"downstairs","session_id":"44450e7d-56fd-4fbc-8233-ff59000df1fb"} {"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:21.272024905Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1237,"":"downstairs","":"downstairs","session_id":"44450e7d-56fd-4fbc-8233-ff59000df1fb"} {"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:21.272034473Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1237,"":"downstairs","":"downstairs","session_id":"44450e7d-56fd-4fbc-8233-ff59000df1fb"} {"msg":"Next flush: 52","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:21.272043832Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1237,"":"downstairs","":"downstairs","session_id":"44450e7d-56fd-4fbc-8233-ff59000df1fb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:21.272053331Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1237,"":"downstairs","":"downstairs","session_id":"44450e7d-56fd-4fbc-8233-ff59000df1fb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:21.272062999Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1237,"session_id":"44450e7d-56fd-4fbc-8233-ff59000df1fb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:21.272071928Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1237,"client":"0","":"downstairs","session_id":"44450e7d-56fd-4fbc-8233-ff59000df1fb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:21.272081637Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1237,"client":"1","":"downstairs","session_id":"44450e7d-56fd-4fbc-8233-ff59000df1fb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:21.272091365Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1237,"client":"2","":"downstairs","session_id":"44450e7d-56fd-4fbc-8233-ff59000df1fb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:21.272101214Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1237,"session_id":"44450e7d-56fd-4fbc-8233-ff59000df1fb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:21.272110332Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1237,"session_id":"44450e7d-56fd-4fbc-8233-ff59000df1fb"} {"msg":"520e6f7f-3caf-4963-82c5-cbe2b8852a4b is now active with session: 44450e7d-56fd-4fbc-8233-ff59000df1fb","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:21.272120441Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1237,"session_id":"44450e7d-56fd-4fbc-8233-ff59000df1fb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:21.27212922Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1237,"session_id":"44450e7d-56fd-4fbc-8233-ff59000df1fb"} {"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:21.272138258Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1237} {"msg":"Activated sub_volume 520e6f7f-3caf-4963-82c5-cbe2b8852a4b","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:21.272147517Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1237} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:21.272155566Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1237,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:21.272441043Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1237,"task":"crutest"} 01/30 Write block 165 len 12288 data: 2 2 2 02/30 Write block 319 len 32768 data: 3 3 3 3 3 3 4 4 03/30 Flush 04/30 Write block 323 len 36864 data: 4 4 5 5 4 3 3 4 4 05/30 Read block 248 len 16384 06/30 Write block 204 len 20480 data: 4 4 4 5 3 07/30 Read block 86 len 28672 08/30 Read block 157 len 36864 09/30 Read block 393 len 8192 10/30 Read block 150 len 32768 11/30 Write block 389 len 28672 data: 3 6 6 6 5 5 6 12/30 Read block 402 len 36864 13/30 Read block 271 len 36864 14/30 Read block 573 len 12288 15/30 Read block 557 len 32768 16/30 Write block 255 len 28672 data: 2 2 3 4 4 3 3 17/30 Flush 18/30 Write block 127 len 12288 data: 3 2 3 19/30 Read block 505 len 12288 20/30 Read block 539 len 24576 21/30 Read block 261 len 32768 22/30 Write block 258 len 24576 data: 5 5 4 4 3 5 23/30 Write block 475 len 24576 data: 2 3 2 3 3 3 24/30 Write block 332 len 20480 data: 4 3 3 3 3 25/30 Write block 461 len 32768 data: 2 3 3 3 3 4 4 4 26/30 Read block 192 len 8192 27/30 Write block 474 len 32768 data: 3 3 4 3 4 4 4 4 28/30 Write block 512 len 12288 data: 3 4 3 29/30 Write block 54 len 20480 data: 3 3 3 2 2 30/30 Read block 384 len 40960 ---------------------------------------------------------------- Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1003 Acked Write 9 Sent Done Done false 1004 Acked Read 4 Sent Done Done false 1005 Acked Write 5 Sent Done Done false 1006 Acked Read 7 Sent Done Done false 1007 Acked Read 9 Sent Done Done false 1008 Acked Read 2 Sent Done Done false 1009 Acked Read 8 Sent Done Done false 1010 Acked Write 7 Sent Done Done false 1011 Acked Read 9 Sent Done Done false 1012 Acked Read 9 Sent Done Done false 1013 Acked Read 3 Sent Done Done false 1014 Acked Read 8 Sent Done Done false 1015 Acked Write 7 Sent Done Done false 1016 Acked Flush 0 Sent Done Done false 1017 Acked Write 3 Sent Done Done false 1018 Acked Read 3 Sent Done Done false 1019 Acked Read 6 Sent Done Done false 1020 Acked Read 8 Sent Done Done false 1021 Acked Write 6 Sent Done Done false 1022 Acked Write 6 Sent Done Done false 1023 Acked Write 5 Sent Done Done false 1024 Acked Write 8 Sent Done Done false 1025 Acked Read 2 Sent Done Done false 1026 Acked Write 8 Sent Done Done false 1027 Acked Write 3 Sent Done Done false 1028 Acked Write 5 Sent Done Done false 1029 Acked Read 10 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 27 0 1 28 Done 0 27 26 53 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1002)) Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:54:21.281493328Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1237} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:21.281662363Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1237,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 18 18 18 33 33 33 F F F 1 020-039 22 22 22 45 45 45 F F F 2 040-059 21 21 21 41 41 41 F T T <--- 3 060-079 18 18 18 35 35 35 F F F 4 080-099 22 22 22 45 45 45 F F F 5 100-119 24 24 24 50 50 50 F F F 6 120-139 24 24 24 50 50 50 F T T <--- 7 140-159 25 25 25 51 51 51 F F F 8 160-179 26 26 26 52 52 52 F F F 9 180-199 24 24 24 50 50 50 F F F 10 200-219 24 26 26 49 53 53 F F F <--- 11 220-239 18 18 18 34 34 34 F F F 12 240-259 17 26 26 31 53 53 F T T <--- 13 260-279 18 26 26 34 53 53 F T T <--- 14 280-299 20 20 20 40 40 40 F F F 15 300-319 26 26 26 52 52 52 F F F 16 320-339 26 26 26 52 53 53 F T T <--- 17 340-359 19 19 19 38 38 38 F F F 18 360-379 20 20 20 40 40 40 F F F 19 380-399 24 26 26 50 53 53 F F F <--- 20 400-419 22 22 22 45 45 45 F F F 21 420-439 25 25 25 51 51 51 F F F 22 440-459 19 19 19 38 38 38 F F F 23 460-479 21 21 21 41 41 41 F T T <--- 24 480-499 22 22 22 45 45 45 F T T <--- 25 500-519 23 23 23 46 46 46 F T T <--- 26 520-539 23 23 23 46 46 46 F F F 27 540-559 18 18 18 35 35 35 F F F 28 560-579 16 16 16 30 30 30 F F F 29 580-599 24 24 24 50 50 50 F F F Max gen: 26, Max flush: 53 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 13 Verifying data now No change after verify Loop: 13 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 18 18 18 33 33 33 F F F 1 020-039 22 22 22 45 45 45 F F F 2 040-059 27 27 27 54 54 54 F F F 3 060-079 18 18 18 35 35 35 F F F 4 080-099 22 22 22 45 45 45 F F F 5 100-119 24 24 24 50 50 50 F F F 6 120-139 27 27 27 54 54 54 F F F 7 140-159 25 25 25 51 51 51 F F F 8 160-179 26 26 26 52 52 52 F F F 9 180-199 24 24 24 50 50 50 F F F 10 200-219 26 26 26 53 53 53 F F F 11 220-239 18 18 18 34 34 34 F F F 12 240-259 27 27 27 54 54 54 F F F 13 260-279 27 27 27 54 54 54 F F F 14 280-299 20 20 20 40 40 40 F F F 15 300-319 26 26 26 52 52 52 F F F 16 320-339 27 27 27 54 54 54 F F F 17 340-359 19 19 19 38 38 38 F F F 18 360-379 20 20 20 40 40 40 F F F 19 380-399 26 26 26 53 53 53 F F F 20 400-419 22 22 22 45 45 45 F F F 21 420-439 25 25 25 51 51 51 F F F 22 440-459 19 19 19 38 38 38 F F F 23 460-479 27 27 27 54 54 54 F F F 24 480-499 27 27 27 54 54 54 F F F 25 500-519 27 27 27 54 54 54 F F F 26 520-539 23 23 23 46 46 46 F F F 27 540-559 18 18 18 35 35 35 F F F 28 560-579 16 16 16 30 30 30 F F F 29 580-599 24 24 24 50 50 50 F F F Max gen: 27, Max flush: 54 Begin loop 14 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:25.53001316Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1245,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:54:25.5371292Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1245,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:25.537184842Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1245,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:25.543880824Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1245,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:25.544162402Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1245} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:25.544187919Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1245} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:25.544196387Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1245} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:25.544207876Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1245} {"msg":"Crucible fc17f0ed-fecc-4c07-ae85-1e56c380700e has session id: 11afeb47-43d8-4a39-83d0-0026b0a39efb","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:25.544573332Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1245,"session_id":"11afeb47-43d8-4a39-83d0-0026b0a39efb"} {"msg":"Upstairs opts: Upstairs UUID: fc17f0ed-fecc-4c07-ae85-1e56c380700e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:25.544595229Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1245,"session_id":"11afeb47-43d8-4a39-83d0-0026b0a39efb"} {"msg":"Crucible stats registered with UUID: fc17f0ed-fecc-4c07-ae85-1e56c380700e","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:25.544610266Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1245,"session_id":"11afeb47-43d8-4a39-83d0-0026b0a39efb"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:54:25.544619595Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1245,"session_id":"11afeb47-43d8-4a39-83d0-0026b0a39efb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:25.544757885Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1245,"":"io task","client":"1","":"downstairs","session_id":"11afeb47-43d8-4a39-83d0-0026b0a39efb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:25.54478924Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1245,"":"io task","client":"0","":"downstairs","session_id":"11afeb47-43d8-4a39-83d0-0026b0a39efb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:25.544800278Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1245,"":"io task","client":"2","":"downstairs","session_id":"11afeb47-43d8-4a39-83d0-0026b0a39efb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:25.54492355Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1245,"task":"crutest"} {"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:25.544942017Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1245} {"msg":"fc17f0ed-fecc-4c07-ae85-1e56c380700e active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:25.544973003Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1245,"session_id":"11afeb47-43d8-4a39-83d0-0026b0a39efb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:25.544982571Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1245,"client":"0","":"downstairs","session_id":"11afeb47-43d8-4a39-83d0-0026b0a39efb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:25.54499226Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1245,"client":"1","":"downstairs","session_id":"11afeb47-43d8-4a39-83d0-0026b0a39efb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:25.545001949Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1245,"client":"2","":"downstairs","session_id":"11afeb47-43d8-4a39-83d0-0026b0a39efb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:25.545011717Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1245,"":"io task","client":"2","":"downstairs","session_id":"11afeb47-43d8-4a39-83d0-0026b0a39efb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:25.545021656Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1245,"":"io task","client":"0","":"downstairs","session_id":"11afeb47-43d8-4a39-83d0-0026b0a39efb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:25.545035164Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1245,"":"io task","client":"1","":"downstairs","session_id":"11afeb47-43d8-4a39-83d0-0026b0a39efb"} {"msg":"ds_connection connected from Ok(127.0.0.1:43372)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:25.545053861Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1245,"":"io task","client":"2","":"downstairs","session_id":"11afeb47-43d8-4a39-83d0-0026b0a39efb"} {"msg":"ds_connection connected from Ok(127.0.0.1:44618)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:25.545064319Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1245,"":"io task","client":"0","":"downstairs","session_id":"11afeb47-43d8-4a39-83d0-0026b0a39efb"} {"msg":"ds_connection connected from Ok(127.0.0.1:54347)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:25.545084326Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1245,"":"io task","client":"1","":"downstairs","session_id":"11afeb47-43d8-4a39-83d0-0026b0a39efb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:25.545832896Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1245,"client":"1","":"downstairs","session_id":"11afeb47-43d8-4a39-83d0-0026b0a39efb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:25.545849524Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1245,"client":"0","":"downstairs","session_id":"11afeb47-43d8-4a39-83d0-0026b0a39efb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:25.545862422Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1245,"client":"2","":"downstairs","session_id":"11afeb47-43d8-4a39-83d0-0026b0a39efb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:25.54614641Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1245,"session_id":"11afeb47-43d8-4a39-83d0-0026b0a39efb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:25.546157968Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1245,"session_id":"11afeb47-43d8-4a39-83d0-0026b0a39efb"} {"msg":"[0]R flush_numbers[0..12]: [33, 45, 54, 35, 45, 50, 54, 51, 52, 50, 53, 34]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:25.546188973Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1245,"":"downstairs","":"downstairs","session_id":"11afeb47-43d8-4a39-83d0-0026b0a39efb"} {"msg":"[0]R generation[0..12]: [18, 22, 27, 18, 22, 24, 27, 25, 26, 24, 26, 18]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:25.546200042Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1245,"":"downstairs","":"downstairs","session_id":"11afeb47-43d8-4a39-83d0-0026b0a39efb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:25.54621089Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1245,"":"downstairs","":"downstairs","session_id":"11afeb47-43d8-4a39-83d0-0026b0a39efb"} {"msg":"[1]R flush_numbers[0..12]: [33, 45, 54, 35, 45, 50, 54, 51, 52, 50, 53, 34]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:25.546221229Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1245,"":"downstairs","":"downstairs","session_id":"11afeb47-43d8-4a39-83d0-0026b0a39efb"} {"msg":"[1]R generation[0..12]: [18, 22, 27, 18, 22, 24, 27, 25, 26, 24, 26, 18]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:25.546231677Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1245,"":"downstairs","":"downstairs","session_id":"11afeb47-43d8-4a39-83d0-0026b0a39efb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:25.546242596Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1245,"":"downstairs","":"downstairs","session_id":"11afeb47-43d8-4a39-83d0-0026b0a39efb"} {"msg":"[2]R flush_numbers[0..12]: [33, 45, 54, 35, 45, 50, 54, 51, 52, 50, 53, 34]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:25.546253064Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1245,"":"downstairs","":"downstairs","session_id":"11afeb47-43d8-4a39-83d0-0026b0a39efb"} {"msg":"[2]R generation[0..12]: [18, 22, 27, 18, 22, 24, 27, 25, 26, 24, 26, 18]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:25.546263432Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1245,"":"downstairs","":"downstairs","session_id":"11afeb47-43d8-4a39-83d0-0026b0a39efb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:25.546274271Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1245,"":"downstairs","":"downstairs","session_id":"11afeb47-43d8-4a39-83d0-0026b0a39efb"} {"msg":"Max found gen is 28","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:25.54628349Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1245,"":"downstairs","":"downstairs","session_id":"11afeb47-43d8-4a39-83d0-0026b0a39efb"} {"msg":"Generation requested: 28 >= found:28","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:25.546293238Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1245,"":"downstairs","":"downstairs","session_id":"11afeb47-43d8-4a39-83d0-0026b0a39efb"} {"msg":"Next flush: 55","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:25.546304176Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1245,"":"downstairs","":"downstairs","session_id":"11afeb47-43d8-4a39-83d0-0026b0a39efb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:25.546313685Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1245,"":"downstairs","":"downstairs","session_id":"11afeb47-43d8-4a39-83d0-0026b0a39efb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:25.546323414Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1245,"session_id":"11afeb47-43d8-4a39-83d0-0026b0a39efb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:25.546332132Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1245,"client":"0","":"downstairs","session_id":"11afeb47-43d8-4a39-83d0-0026b0a39efb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:25.546342001Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1245,"client":"1","":"downstairs","session_id":"11afeb47-43d8-4a39-83d0-0026b0a39efb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:25.546351609Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1245,"client":"2","":"downstairs","session_id":"11afeb47-43d8-4a39-83d0-0026b0a39efb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:25.546361208Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1245,"session_id":"11afeb47-43d8-4a39-83d0-0026b0a39efb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:25.546370277Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1245,"session_id":"11afeb47-43d8-4a39-83d0-0026b0a39efb"} {"msg":"fc17f0ed-fecc-4c07-ae85-1e56c380700e is now active with session: 11afeb47-43d8-4a39-83d0-0026b0a39efb","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:25.546380195Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1245,"session_id":"11afeb47-43d8-4a39-83d0-0026b0a39efb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:25.546389074Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1245,"session_id":"11afeb47-43d8-4a39-83d0-0026b0a39efb"} {"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:25.546398173Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1245} {"msg":"Activated sub_volume fc17f0ed-fecc-4c07-ae85-1e56c380700e","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:25.546406511Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1245} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:25.54641475Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1245,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:25.546695479Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1245,"task":"crutest"} 01/30 Write block 493 len 20480 data: 2 2 2 2 2 02/30 Write block 78 len 12288 data: 3 2 2 03/30 Write block 111 len 20480 data: 2 2 2 2 2 04/30 Write block 561 len 20480 data: 4 3 3 3 3 05/30 Read block 320 len 32768 06/30 Flush 07/30 Write block 15 len 32768 data: 4 4 4 4 4 4 4 5 08/30 Write block 115 len 36864 data: 3 2 2 2 2 2 2 2 2 09/30 Read block 340 len 28672 10/30 Read block 288 len 8192 11/30 Flush 12/30 Write block 544 len 12288 data: 3 4 3 13/30 Read block 446 len 28672 14/30 Write block 515 len 4096 data: 3 15/30 Read block 313 len 20480 16/30 Write block 283 len 16384 data: 2 4 4 6 17/30 Write block 363 len 16384 data: 4 4 4 3 18/30 Write block 510 len 16384 data: 4 3 4 5 19/30 Write block 60 len 36864 data: 2 2 3 2 2 2 2 3 3 20/30 Read block 329 len 32768 21/30 Write block 441 len 36864 data: 3 3 3 6 6 6 5 5 4 22/30 Write block 581 len 4096 data: 5 23/30 Read block 393 len 12288 24/30 Read block 212 len 16384 25/30 Read block 156 len 12288 26/30 Read block 477 len 28672 27/30 Write block 138 len 36864 data: 6 5 5 5 4 4 5 5 4 28/30 Read block 104 len 8192 29/30 Write block 31 len 24576 data: 4 4 3 4 5 5 30/30 Read block 175 len 40960 ---------------------------------------------------------------- Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 8 Done Done Done false 1007 Acked Write 9 Done Done Done false 1008 Acked Read 7 Done Done Done false 1009 Acked Read 2 Done Done Done false 1010 Acked Flush 0 Sent Done Done false 1011 Acked Write 3 Sent Done Done false 1012 Acked Read 7 Sent Done Done false 1013 Acked Write 1 Sent Done Done false 1014 Acked Read 5 Sent Done Done false 1015 Acked Write 4 Sent Done Done false 1016 Acked Write 4 Sent Done Done false 1017 Acked Write 4 Sent Done Done false 1018 Acked Write 9 Sent Done Done false 1019 Acked Read 8 Sent Done Done false 1020 Acked Write 9 Sent Done Done false 1021 Acked Write 1 Sent Done Done false 1022 Acked Read 3 Sent Done Done false 1023 Acked Read 4 Sent Done Done false 1024 Acked Read 3 Sent Done Done false 1025 Acked Read 7 Sent Done Done false 1026 Acked Write 9 Sent Done Done false 1027 Acked Read 2 Sent Done Done false 1028 Acked Write 6 Sent Done Done false 1029 Acked Read 10 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 20 1 0 21 Done 4 23 24 51 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1005)) Some(JobId(1010)) Some(JobId(1010)) Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:54:25.554792904Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1245} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:25.554964489Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1245,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 18 28 28 33 56 56 T F F <--- 1 020-039 22 28 28 45 56 56 T T T <--- 2 040-059 27 27 27 54 54 54 F F F 3 060-079 28 28 28 55 55 55 F T T <--- 4 080-099 28 28 28 55 55 55 F F F 5 100-119 28 28 28 55 56 56 T F F <--- 6 120-139 27 28 28 54 56 56 T T T <--- 7 140-159 25 25 25 51 51 51 F T T <--- 8 160-179 26 26 26 52 52 52 F F F 9 180-199 24 24 24 50 50 50 F F F 10 200-219 26 26 26 53 53 53 F F F 11 220-239 18 18 18 34 34 34 F F F 12 240-259 27 27 27 54 54 54 F F F 13 260-279 27 27 27 54 54 54 F F F 14 280-299 20 20 20 40 40 40 F T T <--- 15 300-319 26 26 26 52 52 52 F F F 16 320-339 27 27 27 54 54 54 F F F 17 340-359 19 19 19 38 38 38 F F F 18 360-379 20 20 20 40 40 40 F T T <--- 19 380-399 26 26 26 53 53 53 F F F 20 400-419 22 22 22 45 45 45 F F F 21 420-439 25 25 25 51 51 51 F F F 22 440-459 19 19 19 38 38 38 F T T <--- 23 460-479 27 27 27 54 54 54 F F F 24 480-499 28 28 28 55 55 55 F F F 25 500-519 27 27 27 54 54 54 F T T <--- 26 520-539 23 23 23 46 46 46 F F F 27 540-559 18 18 18 35 35 35 F T T <--- 28 560-579 28 28 28 55 55 55 F F F 29 580-599 24 24 24 50 50 50 F T T <--- Max gen: 28, Max flush: 56 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 14 Verifying data now No change after verify Loop: 14 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 28 28 28 56 56 56 F F F 1 020-039 29 29 29 57 57 57 F F F 2 040-059 27 27 27 54 54 54 F F F 3 060-079 29 29 29 57 57 57 F F F 4 080-099 28 28 28 55 55 55 F F F 5 100-119 28 28 28 56 56 56 F F F 6 120-139 29 29 29 57 57 57 F F F 7 140-159 29 29 29 57 57 57 F F F 8 160-179 26 26 26 52 52 52 F F F 9 180-199 24 24 24 50 50 50 F F F 10 200-219 26 26 26 53 53 53 F F F 11 220-239 18 18 18 34 34 34 F F F 12 240-259 27 27 27 54 54 54 F F F 13 260-279 27 27 27 54 54 54 F F F 14 280-299 29 29 29 57 57 57 F F F 15 300-319 26 26 26 52 52 52 F F F 16 320-339 27 27 27 54 54 54 F F F 17 340-359 19 19 19 38 38 38 F F F 18 360-379 29 29 29 57 57 57 F F F 19 380-399 26 26 26 53 53 53 F F F 20 400-419 22 22 22 45 45 45 F F F 21 420-439 25 25 25 51 51 51 F F F 22 440-459 29 29 29 57 57 57 F F F 23 460-479 27 27 27 54 54 54 F F F 24 480-499 28 28 28 55 55 55 F F F 25 500-519 29 29 29 57 57 57 F F F 26 520-539 23 23 23 46 46 46 F F F 27 540-559 29 29 29 57 57 57 F F F 28 560-579 28 28 28 55 55 55 F F F 29 580-599 29 29 29 57 57 57 F F F Max gen: 29, Max flush: 57 Begin loop 15 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:29.809292344Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1253,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:54:29.816386747Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1253,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:29.816442379Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1253,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:29.823203951Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1253,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:29.823468252Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1253} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:29.823493168Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1253} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:29.823501637Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1253} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:29.823513145Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1253} {"msg":"Crucible 7aa1b683-a38e-4e17-8913-506dda9d654a has session id: 05687a4e-07f5-44e0-9e8d-c96594fae1a8","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:29.823864283Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1253,"session_id":"05687a4e-07f5-44e0-9e8d-c96594fae1a8"} {"msg":"Upstairs opts: Upstairs UUID: 7aa1b683-a38e-4e17-8913-506dda9d654a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:29.82388743Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1253,"session_id":"05687a4e-07f5-44e0-9e8d-c96594fae1a8"} {"msg":"Crucible stats registered with UUID: 7aa1b683-a38e-4e17-8913-506dda9d654a","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:29.823897598Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1253,"session_id":"05687a4e-07f5-44e0-9e8d-c96594fae1a8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:54:29.823906677Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1253,"session_id":"05687a4e-07f5-44e0-9e8d-c96594fae1a8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:29.824058765Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1253,"":"io task","client":"0","":"downstairs","session_id":"05687a4e-07f5-44e0-9e8d-c96594fae1a8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:29.824075912Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1253,"":"io task","client":"1","":"downstairs","session_id":"05687a4e-07f5-44e0-9e8d-c96594fae1a8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:29.824086121Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1253,"":"io task","client":"2","":"downstairs","session_id":"05687a4e-07f5-44e0-9e8d-c96594fae1a8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:29.824251296Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1253,"task":"crutest"} {"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:29.824266214Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1253} {"msg":"7aa1b683-a38e-4e17-8913-506dda9d654a active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:29.824289581Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1253,"session_id":"05687a4e-07f5-44e0-9e8d-c96594fae1a8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:29.824298909Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1253,"client":"0","":"downstairs","session_id":"05687a4e-07f5-44e0-9e8d-c96594fae1a8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:29.824310078Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1253,"client":"1","":"downstairs","session_id":"05687a4e-07f5-44e0-9e8d-c96594fae1a8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:29.824319766Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1253,"client":"2","":"downstairs","session_id":"05687a4e-07f5-44e0-9e8d-c96594fae1a8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:29.824329395Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1253,"":"io task","client":"2","":"downstairs","session_id":"05687a4e-07f5-44e0-9e8d-c96594fae1a8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:29.824339503Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1253,"":"io task","client":"0","":"downstairs","session_id":"05687a4e-07f5-44e0-9e8d-c96594fae1a8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:29.824349562Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1253,"":"io task","client":"1","":"downstairs","session_id":"05687a4e-07f5-44e0-9e8d-c96594fae1a8"} {"msg":"ds_connection connected from Ok(127.0.0.1:52533)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:29.824385906Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1253,"":"io task","client":"0","":"downstairs","session_id":"05687a4e-07f5-44e0-9e8d-c96594fae1a8"} {"msg":"ds_connection connected from Ok(127.0.0.1:48153)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:29.824396325Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1253,"":"io task","client":"2","":"downstairs","session_id":"05687a4e-07f5-44e0-9e8d-c96594fae1a8"} {"msg":"ds_connection connected from Ok(127.0.0.1:51276)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:29.824406783Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1253,"":"io task","client":"1","":"downstairs","session_id":"05687a4e-07f5-44e0-9e8d-c96594fae1a8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:29.825143415Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1253,"client":"1","":"downstairs","session_id":"05687a4e-07f5-44e0-9e8d-c96594fae1a8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:29.825162842Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1253,"client":"0","":"downstairs","session_id":"05687a4e-07f5-44e0-9e8d-c96594fae1a8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:29.82517728Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1253,"client":"2","":"downstairs","session_id":"05687a4e-07f5-44e0-9e8d-c96594fae1a8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:29.825478245Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1253,"session_id":"05687a4e-07f5-44e0-9e8d-c96594fae1a8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:29.825487944Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1253,"session_id":"05687a4e-07f5-44e0-9e8d-c96594fae1a8"} {"msg":"[0]R flush_numbers[0..12]: [56, 57, 54, 57, 55, 56, 57, 57, 52, 50, 53, 34]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:29.825521229Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1253,"":"downstairs","":"downstairs","session_id":"05687a4e-07f5-44e0-9e8d-c96594fae1a8"} {"msg":"[0]R generation[0..12]: [28, 29, 27, 29, 28, 28, 29, 29, 26, 24, 26, 18]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:29.825532287Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1253,"":"downstairs","":"downstairs","session_id":"05687a4e-07f5-44e0-9e8d-c96594fae1a8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:29.825545215Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1253,"":"downstairs","":"downstairs","session_id":"05687a4e-07f5-44e0-9e8d-c96594fae1a8"} {"msg":"[1]R flush_numbers[0..12]: [56, 57, 54, 57, 55, 56, 57, 57, 52, 50, 53, 34]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:29.825555584Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1253,"":"downstairs","":"downstairs","session_id":"05687a4e-07f5-44e0-9e8d-c96594fae1a8"} {"msg":"[1]R generation[0..12]: [28, 29, 27, 29, 28, 28, 29, 29, 26, 24, 26, 18]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:29.825566012Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1253,"":"downstairs","":"downstairs","session_id":"05687a4e-07f5-44e0-9e8d-c96594fae1a8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:29.825577061Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1253,"":"downstairs","":"downstairs","session_id":"05687a4e-07f5-44e0-9e8d-c96594fae1a8"} {"msg":"[2]R flush_numbers[0..12]: [56, 57, 54, 57, 55, 56, 57, 57, 52, 50, 53, 34]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:29.825587609Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1253,"":"downstairs","":"downstairs","session_id":"05687a4e-07f5-44e0-9e8d-c96594fae1a8"} {"msg":"[2]R generation[0..12]: [28, 29, 27, 29, 28, 28, 29, 29, 26, 24, 26, 18]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:29.825598048Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1253,"":"downstairs","":"downstairs","session_id":"05687a4e-07f5-44e0-9e8d-c96594fae1a8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:29.825608716Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1253,"":"downstairs","":"downstairs","session_id":"05687a4e-07f5-44e0-9e8d-c96594fae1a8"} {"msg":"Max found gen is 30","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:29.825618055Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1253,"":"downstairs","":"downstairs","session_id":"05687a4e-07f5-44e0-9e8d-c96594fae1a8"} {"msg":"Generation requested: 30 >= found:30","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:29.825627623Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1253,"":"downstairs","":"downstairs","session_id":"05687a4e-07f5-44e0-9e8d-c96594fae1a8"} {"msg":"Next flush: 58","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:29.825636912Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1253,"":"downstairs","":"downstairs","session_id":"05687a4e-07f5-44e0-9e8d-c96594fae1a8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:29.82564645Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1253,"":"downstairs","":"downstairs","session_id":"05687a4e-07f5-44e0-9e8d-c96594fae1a8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:29.825656379Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1253,"session_id":"05687a4e-07f5-44e0-9e8d-c96594fae1a8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:29.825665148Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1253,"client":"0","":"downstairs","session_id":"05687a4e-07f5-44e0-9e8d-c96594fae1a8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:29.825674876Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1253,"client":"1","":"downstairs","session_id":"05687a4e-07f5-44e0-9e8d-c96594fae1a8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:29.825684515Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1253,"client":"2","":"downstairs","session_id":"05687a4e-07f5-44e0-9e8d-c96594fae1a8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:29.825695383Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1253,"session_id":"05687a4e-07f5-44e0-9e8d-c96594fae1a8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:29.825704422Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1253,"session_id":"05687a4e-07f5-44e0-9e8d-c96594fae1a8"} {"msg":"7aa1b683-a38e-4e17-8913-506dda9d654a is now active with session: 05687a4e-07f5-44e0-9e8d-c96594fae1a8","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:29.82571437Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1253,"session_id":"05687a4e-07f5-44e0-9e8d-c96594fae1a8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:29.825723109Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1253,"session_id":"05687a4e-07f5-44e0-9e8d-c96594fae1a8"} {"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:29.825732178Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1253} {"msg":"Activated sub_volume 7aa1b683-a38e-4e17-8913-506dda9d654a","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:29.825740217Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1253} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:29.825748395Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1253,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:29.826041702Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1253,"task":"crutest"} 01/30 Write block 190 len 20480 data: 3 5 5 5 5 02/30 Write block 203 len 24576 data: 4 5 5 5 6 4 03/30 Read block 168 len 28672 04/30 Read block 159 len 8192 05/30 Read block 287 len 36864 06/30 Flush 07/30 Write block 538 len 28672 data: 2 2 2 2 2 3 4 08/30 Write block 456 len 16384 data: 2 3 3 3 09/30 Flush 10/30 Write block 84 len 32768 data: 3 4 4 4 6 5 6 5 11/30 Write block 39 len 32768 data: 7 7 7 6 7 6 5 5 12/30 Read block 33 len 16384 13/30 Read block 94 len 16384 14/30 Write block 390 len 4096 data: 7 15/30 Read block 318 len 32768 16/30 Read block 402 len 24576 17/30 Read block 312 len 4096 18/30 Read block 354 len 32768 19/30 Read block 575 len 20480 20/30 Write block 514 len 8192 data: 4 4 21/30 Write block 16 len 28672 data: 5 5 5 5 5 5 6 22/30 Read block 530 len 4096 23/30 Read block 181 len 8192 24/30 Write block 219 len 16384 data: 2 3 3 3 25/30 Write block 581 len 4096 data: 6 26/30 Flush 27/30 Write block 47 len 20480 data: 5 4 3 3 3 28/30 Read block 69 len 28672 29/30 Write block 290 len 32768 data: 3 3 3 3 2 2 2 2 30/30 Read block 292 len 28672 ---------------------------------------------------------------- Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Write 6 Done Done Done false 1002 Acked Read 7 Done Done Done false 1003 Acked Read 2 Sent Done Done false 1004 Acked Read 9 Sent Done Done false 1005 Acked Flush 0 Sent Done Done false 1006 Acked Write 7 Sent Done Done false 1007 Acked Write 4 Sent Done Done false 1008 Acked Flush 0 Sent Done Done false 1009 Acked Write 8 Sent Done Done false 1010 Acked Write 8 Sent Done Done false 1011 Acked Read 4 Sent Done Done false 1012 Acked Read 4 Sent Done Done false 1013 Acked Write 1 Sent Done Done false 1014 Acked Read 8 Sent Done Done false 1015 Acked Read 6 Sent Done Done false 1016 Acked Read 1 Sent Done Done false 1017 Acked Read 8 Sent Done Done false 1018 Acked Read 5 Sent Done Done false 1019 Acked Write 2 Sent Done Done false 1020 Acked Write 7 Sent Done Done false 1021 Acked Read 1 Sent Done Done false 1022 Acked Read 2 Sent Done Done false 1023 Acked Write 4 Sent Done Done false 1024 Acked Write 1 Sent Done Done false 1025 Acked Flush 0 Sent Done Done false 1026 Acked Write 5 Sent Done Done false 1027 Acked Read 7 Sent Done Done false 1028 Acked Write 8 Sent Done Done false 1029 Acked Read 7 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 27 0 1 28 Done 3 30 29 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:54:29.834632005Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1253} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:29.834809009Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1253,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 28 30 30 56 60 60 F F F <--- 1 020-039 29 30 30 57 60 60 F F F <--- 2 040-059 27 30 30 54 60 60 F T T <--- 3 060-079 29 29 29 57 57 57 F F F 4 080-099 28 30 30 55 60 60 F F F <--- 5 100-119 28 28 28 56 56 56 F F F 6 120-139 29 29 29 57 57 57 F F F 7 140-159 29 29 29 57 57 57 F F F 8 160-179 26 26 26 52 52 52 F F F 9 180-199 24 30 30 50 58 58 T F F <--- 10 200-219 26 30 30 53 60 60 T F F <--- 11 220-239 18 30 30 34 60 60 F F F <--- 12 240-259 27 27 27 54 54 54 F F F 13 260-279 27 27 27 54 54 54 F F F 14 280-299 29 29 29 57 57 57 F T T <--- 15 300-319 26 26 26 52 52 52 F F F 16 320-339 27 27 27 54 54 54 F F F 17 340-359 19 19 19 38 38 38 F F F 18 360-379 29 29 29 57 57 57 F F F 19 380-399 26 30 30 53 60 60 F F F <--- 20 400-419 22 22 22 45 45 45 F F F 21 420-439 25 25 25 51 51 51 F F F 22 440-459 29 30 30 57 59 59 F F F <--- 23 460-479 27 27 27 54 54 54 F F F 24 480-499 28 28 28 55 55 55 F F F 25 500-519 29 30 30 57 60 60 F F F <--- 26 520-539 23 30 30 46 59 59 F F F <--- 27 540-559 29 30 30 57 59 59 F F F <--- 28 560-579 28 28 28 55 55 55 F F F 29 580-599 29 30 30 57 60 60 F F F <--- Max gen: 30, Max flush: 60 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 15 Verifying data now No change after verify Loop: 15 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 30 30 30 60 60 60 F F F 1 020-039 30 30 30 60 60 60 F F F 2 040-059 31 31 31 61 61 61 F F F 3 060-079 29 29 29 57 57 57 F F F 4 080-099 30 30 30 60 60 60 F F F 5 100-119 28 28 28 56 56 56 F F F 6 120-139 29 29 29 57 57 57 F F F 7 140-159 29 29 29 57 57 57 F F F 8 160-179 26 26 26 52 52 52 F F F 9 180-199 30 30 30 58 58 58 F F F 10 200-219 30 30 30 60 60 60 F F F 11 220-239 30 30 30 60 60 60 F F F 12 240-259 27 27 27 54 54 54 F F F 13 260-279 27 27 27 54 54 54 F F F 14 280-299 31 31 31 61 61 61 F F F 15 300-319 26 26 26 52 52 52 F F F 16 320-339 27 27 27 54 54 54 F F F 17 340-359 19 19 19 38 38 38 F F F 18 360-379 29 29 29 57 57 57 F F F 19 380-399 30 30 30 60 60 60 F F F 20 400-419 22 22 22 45 45 45 F F F 21 420-439 25 25 25 51 51 51 F F F 22 440-459 30 30 30 59 59 59 F F F 23 460-479 27 27 27 54 54 54 F F F 24 480-499 28 28 28 55 55 55 F F F 25 500-519 30 30 30 60 60 60 F F F 26 520-539 30 30 30 59 59 59 F F F 27 540-559 30 30 30 59 59 59 F F F 28 560-579 28 28 28 55 55 55 F F F 29 580-599 30 30 30 60 60 60 F F F Max gen: 31, Max flush: 61 Begin loop 16 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:34.08977371Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1261,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:54:34.096852225Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1261,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:34.096907787Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1261,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:34.103643174Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1261,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:34.103902495Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1261} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:34.103926432Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1261} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:34.10393507Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1261} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:34.103946769Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1261} {"msg":"Crucible 62105e11-4308-476b-b8ad-381658d2a342 has session id: fe8b2391-7d7a-4a50-95b3-dd8cf61f642f","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:34.104337531Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1261,"session_id":"fe8b2391-7d7a-4a50-95b3-dd8cf61f642f"} {"msg":"Upstairs opts: Upstairs UUID: 62105e11-4308-476b-b8ad-381658d2a342, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:34.104366587Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1261,"session_id":"fe8b2391-7d7a-4a50-95b3-dd8cf61f642f"} {"msg":"Crucible stats registered with UUID: 62105e11-4308-476b-b8ad-381658d2a342","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:34.104377515Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1261,"session_id":"fe8b2391-7d7a-4a50-95b3-dd8cf61f642f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:54:34.104386704Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1261,"session_id":"fe8b2391-7d7a-4a50-95b3-dd8cf61f642f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:34.104507406Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1261,"":"io task","client":"0","":"downstairs","session_id":"fe8b2391-7d7a-4a50-95b3-dd8cf61f642f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:34.104523764Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1261,"":"io task","client":"2","":"downstairs","session_id":"fe8b2391-7d7a-4a50-95b3-dd8cf61f642f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:34.104533972Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1261,"":"io task","client":"1","":"downstairs","session_id":"fe8b2391-7d7a-4a50-95b3-dd8cf61f642f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:34.104689599Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1261,"task":"crutest"} {"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:34.104705527Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1261} {"msg":"62105e11-4308-476b-b8ad-381658d2a342 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:34.104728823Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1261,"session_id":"fe8b2391-7d7a-4a50-95b3-dd8cf61f642f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:34.104738272Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1261,"client":"0","":"downstairs","session_id":"fe8b2391-7d7a-4a50-95b3-dd8cf61f642f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:34.10474808Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1261,"client":"1","":"downstairs","session_id":"fe8b2391-7d7a-4a50-95b3-dd8cf61f642f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:34.104758059Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1261,"client":"2","":"downstairs","session_id":"fe8b2391-7d7a-4a50-95b3-dd8cf61f642f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:34.104767678Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1261,"":"io task","client":"2","":"downstairs","session_id":"fe8b2391-7d7a-4a50-95b3-dd8cf61f642f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:34.104777796Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1261,"":"io task","client":"0","":"downstairs","session_id":"fe8b2391-7d7a-4a50-95b3-dd8cf61f642f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:34.104788075Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1261,"":"io task","client":"1","":"downstairs","session_id":"fe8b2391-7d7a-4a50-95b3-dd8cf61f642f"} {"msg":"ds_connection connected from Ok(127.0.0.1:62721)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:34.104845656Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1261,"":"io task","client":"2","":"downstairs","session_id":"fe8b2391-7d7a-4a50-95b3-dd8cf61f642f"} {"msg":"ds_connection connected from Ok(127.0.0.1:45062)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:34.104856664Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1261,"":"io task","client":"1","":"downstairs","session_id":"fe8b2391-7d7a-4a50-95b3-dd8cf61f642f"} {"msg":"ds_connection connected from Ok(127.0.0.1:58594)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:34.104946611Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1261,"":"io task","client":"0","":"downstairs","session_id":"fe8b2391-7d7a-4a50-95b3-dd8cf61f642f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:34.105636989Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1261,"client":"1","":"downstairs","session_id":"fe8b2391-7d7a-4a50-95b3-dd8cf61f642f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:34.105659966Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1261,"client":"2","":"downstairs","session_id":"fe8b2391-7d7a-4a50-95b3-dd8cf61f642f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:34.105671384Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1261,"client":"0","":"downstairs","session_id":"fe8b2391-7d7a-4a50-95b3-dd8cf61f642f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:34.105967881Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1261,"session_id":"fe8b2391-7d7a-4a50-95b3-dd8cf61f642f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:34.105977369Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1261,"session_id":"fe8b2391-7d7a-4a50-95b3-dd8cf61f642f"} {"msg":"[0]R flush_numbers[0..12]: [60, 60, 61, 57, 60, 56, 57, 57, 52, 58, 60, 60]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:34.106014834Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1261,"":"downstairs","":"downstairs","session_id":"fe8b2391-7d7a-4a50-95b3-dd8cf61f642f"} {"msg":"[0]R generation[0..12]: [30, 30, 31, 29, 30, 28, 29, 29, 26, 30, 30, 30]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:34.106025872Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1261,"":"downstairs","":"downstairs","session_id":"fe8b2391-7d7a-4a50-95b3-dd8cf61f642f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:34.10603686Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1261,"":"downstairs","":"downstairs","session_id":"fe8b2391-7d7a-4a50-95b3-dd8cf61f642f"} {"msg":"[1]R flush_numbers[0..12]: [60, 60, 61, 57, 60, 56, 57, 57, 52, 58, 60, 60]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:34.106047329Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1261,"":"downstairs","":"downstairs","session_id":"fe8b2391-7d7a-4a50-95b3-dd8cf61f642f"} {"msg":"[1]R generation[0..12]: [30, 30, 31, 29, 30, 28, 29, 29, 26, 30, 30, 30]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:34.106057857Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1261,"":"downstairs","":"downstairs","session_id":"fe8b2391-7d7a-4a50-95b3-dd8cf61f642f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:34.106069046Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1261,"":"downstairs","":"downstairs","session_id":"fe8b2391-7d7a-4a50-95b3-dd8cf61f642f"} {"msg":"[2]R flush_numbers[0..12]: [60, 60, 61, 57, 60, 56, 57, 57, 52, 58, 60, 60]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:34.106081634Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1261,"":"downstairs","":"downstairs","session_id":"fe8b2391-7d7a-4a50-95b3-dd8cf61f642f"} {"msg":"[2]R generation[0..12]: [30, 30, 31, 29, 30, 28, 29, 29, 26, 30, 30, 30]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:34.106091912Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1261,"":"downstairs","":"downstairs","session_id":"fe8b2391-7d7a-4a50-95b3-dd8cf61f642f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:34.106102721Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1261,"":"downstairs","":"downstairs","session_id":"fe8b2391-7d7a-4a50-95b3-dd8cf61f642f"} {"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:34.106112249Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1261,"":"downstairs","":"downstairs","session_id":"fe8b2391-7d7a-4a50-95b3-dd8cf61f642f"} {"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:34.106121908Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1261,"":"downstairs","":"downstairs","session_id":"fe8b2391-7d7a-4a50-95b3-dd8cf61f642f"} {"msg":"Next flush: 62","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:34.106131216Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1261,"":"downstairs","":"downstairs","session_id":"fe8b2391-7d7a-4a50-95b3-dd8cf61f642f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:34.106140475Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1261,"":"downstairs","":"downstairs","session_id":"fe8b2391-7d7a-4a50-95b3-dd8cf61f642f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:34.106150424Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1261,"session_id":"fe8b2391-7d7a-4a50-95b3-dd8cf61f642f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:34.106159182Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1261,"client":"0","":"downstairs","session_id":"fe8b2391-7d7a-4a50-95b3-dd8cf61f642f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:34.106168871Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1261,"client":"1","":"downstairs","session_id":"fe8b2391-7d7a-4a50-95b3-dd8cf61f642f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:34.106178599Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1261,"client":"2","":"downstairs","session_id":"fe8b2391-7d7a-4a50-95b3-dd8cf61f642f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:34.106188568Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1261,"session_id":"fe8b2391-7d7a-4a50-95b3-dd8cf61f642f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:34.106197737Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1261,"session_id":"fe8b2391-7d7a-4a50-95b3-dd8cf61f642f"} {"msg":"62105e11-4308-476b-b8ad-381658d2a342 is now active with session: fe8b2391-7d7a-4a50-95b3-dd8cf61f642f","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:34.106207695Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1261,"session_id":"fe8b2391-7d7a-4a50-95b3-dd8cf61f642f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:34.106216434Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1261,"session_id":"fe8b2391-7d7a-4a50-95b3-dd8cf61f642f"} {"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:34.106226602Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1261} {"msg":"Activated sub_volume 62105e11-4308-476b-b8ad-381658d2a342","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:34.106234561Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1261} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:34.10624269Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1261,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:34.106535787Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1261,"task":"crutest"} 01/30 Write block 38 len 8192 data: 7 8 02/30 Flush 03/30 Write block 311 len 40960 data: 7 7 7 7 7 6 4 3 4 4 04/30 Read block 58 len 36864 05/30 Read block 349 len 40960 06/30 Write block 139 len 24576 data: 6 6 6 5 5 6 07/30 Write block 26 len 12288 data: 5 5 5 08/30 Write block 184 len 4096 data: 3 09/30 Write block 245 len 40960 data: 5 5 4 4 3 3 2 2 2 2 10/30 Write block 557 len 32768 data: 5 5 3 3 5 4 4 4 11/30 Read block 232 len 28672 12/30 Write block 153 len 8192 data: 4 4 13/30 Read block 80 len 8192 14/30 Read block 374 len 16384 15/30 Write block 515 len 32768 data: 5 3 4 6 5 4 4 4 16/30 Write block 363 len 16384 data: 5 5 5 4 17/30 Read block 148 len 20480 18/30 Read block 114 len 20480 19/30 Write block 402 len 24576 data: 2 2 2 2 3 3 20/30 Write block 274 len 12288 data: 3 4 4 21/30 Write block 337 len 28672 data: 3 3 4 3 2 2 2 22/30 Write block 147 len 8192 data: 4 4 23/30 Flush 24/30 Write block 411 len 24576 data: 7 7 6 5 6 7 25/30 Read block 276 len 36864 26/30 Write block 354 len 8192 data: 3 3 27/30 Write block 66 len 36864 data: 3 4 4 3 3 3 3 3 3 28/30 Write block 145 len 40960 data: 6 5 5 5 4 4 4 4 5 5 29/30 Read block 146 len 32768 30/30 Read block 10 len 40960 ---------------------------------------------------------------- Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 10 Done Done Done false 1003 Acked Read 9 Done Done Done false 1004 Acked Read 10 Sent Done Done false 1005 Acked Write 6 Sent Done Done false 1006 Acked Write 3 Sent Done Done false 1007 Acked Write 1 Sent Done Done false 1008 Acked Write 10 Sent Done Done false 1009 Acked Write 8 Sent Done Done false 1010 Acked Read 7 Sent Done Done false 1011 Acked Write 2 Sent Done Done false 1012 Acked Read 2 Sent Done Done false 1013 Acked Read 4 Sent Done Done false 1014 Acked Write 8 Sent Done Done false 1015 Acked Write 4 Sent Done Done false 1016 Acked Read 5 Sent Done Done false 1017 Acked Read 5 Sent Done Done false 1018 Acked Write 6 Sent Done Done false 1019 Acked Write 3 Sent Done Done false 1020 Acked Write 7 Sent Done Done false 1021 Acked Write 2 Sent Done Done false 1022 Acked Flush 0 Sent Done Done false 1023 Acked Write 6 Sent Done Done false 1024 Acked Read 9 Sent Done Done false 1025 Acked Write 2 Sent Done Done false 1026 Acked Write 9 Sent Done Done false 1027 Acked Write 10 Sent Done Done false 1028 Acked Read 8 Sent Done Done false 1029 Acked Read 10 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 26 0 0 26 Done 2 28 28 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1001)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:54:34.115365304Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1261} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:34.115539388Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1261,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 30 30 30 60 60 60 F F F 1 020-039 32 32 32 62 63 63 F F F <--- 2 040-059 31 31 31 61 61 61 F F F 3 060-079 29 29 29 57 57 57 F T T <--- 4 080-099 30 30 30 60 60 60 F F F 5 100-119 28 28 28 56 56 56 F F F 6 120-139 29 32 32 57 63 63 F F F <--- 7 140-159 29 32 32 57 63 63 F T T <--- 8 160-179 26 26 26 52 52 52 F F F 9 180-199 30 32 32 58 63 63 F F F <--- 10 200-219 30 30 30 60 60 60 F F F 11 220-239 30 30 30 60 60 60 F F F 12 240-259 27 32 32 54 63 63 F F F <--- 13 260-279 27 32 32 54 63 63 F F F <--- 14 280-299 31 31 31 61 61 61 F F F 15 300-319 26 32 32 52 63 63 T F F <--- 16 320-339 27 32 32 54 63 63 T F F <--- 17 340-359 19 32 32 38 63 63 F T T <--- 18 360-379 29 32 32 57 63 63 F F F <--- 19 380-399 30 30 30 60 60 60 F F F 20 400-419 22 32 32 45 63 63 F T T <--- 21 420-439 25 25 25 51 51 51 F F F 22 440-459 30 30 30 59 59 59 F F F 23 460-479 27 27 27 54 54 54 F F F 24 480-499 28 28 28 55 55 55 F F F 25 500-519 30 32 32 60 63 63 F F F <--- 26 520-539 30 32 32 59 63 63 F F F <--- 27 540-559 30 32 32 59 63 63 F F F <--- 28 560-579 28 32 32 55 63 63 F F F <--- 29 580-599 30 30 30 60 60 60 F F F Max gen: 32, Max flush: 63 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 16 Verifying data now No change after verify Loop: 16 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 30 30 30 60 60 60 F F F 1 020-039 32 32 32 63 63 63 F F F 2 040-059 31 31 31 61 61 61 F F F 3 060-079 33 33 33 64 64 64 F F F 4 080-099 30 30 30 60 60 60 F F F 5 100-119 28 28 28 56 56 56 F F F 6 120-139 32 32 32 63 63 63 F F F 7 140-159 33 33 33 64 64 64 F F F 8 160-179 26 26 26 52 52 52 F F F 9 180-199 32 32 32 63 63 63 F F F 10 200-219 30 30 30 60 60 60 F F F 11 220-239 30 30 30 60 60 60 F F F 12 240-259 32 32 32 63 63 63 F F F 13 260-279 32 32 32 63 63 63 F F F 14 280-299 31 31 31 61 61 61 F F F 15 300-319 32 32 32 63 63 63 F F F 16 320-339 32 32 32 63 63 63 F F F 17 340-359 33 33 33 64 64 64 F F F 18 360-379 32 32 32 63 63 63 F F F 19 380-399 30 30 30 60 60 60 F F F 20 400-419 33 33 33 64 64 64 F F F 21 420-439 25 25 25 51 51 51 F F F 22 440-459 30 30 30 59 59 59 F F F 23 460-479 27 27 27 54 54 54 F F F 24 480-499 28 28 28 55 55 55 F F F 25 500-519 32 32 32 63 63 63 F F F 26 520-539 32 32 32 63 63 63 F F F 27 540-559 32 32 32 63 63 63 F F F 28 560-579 32 32 32 63 63 63 F F F 29 580-599 30 30 30 60 60 60 F F F Max gen: 33, Max flush: 64 Begin loop 17 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:38.374112137Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1269,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:54:38.381174575Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1269,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:38.381229117Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1269,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:38.388016496Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1269,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:38.388280127Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1269} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:38.388305173Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1269} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:38.388313692Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1269} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:38.38832528Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1269} {"msg":"Crucible 0a8c98b3-403d-4b9f-810a-8ee27583193a has session id: 25d3630b-f888-46e0-9a49-9b647c10c399","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:38.388640964Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1269,"session_id":"25d3630b-f888-46e0-9a49-9b647c10c399"} {"msg":"Upstairs opts: Upstairs UUID: 0a8c98b3-403d-4b9f-810a-8ee27583193a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:38.388662601Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1269,"session_id":"25d3630b-f888-46e0-9a49-9b647c10c399"} {"msg":"Crucible stats registered with UUID: 0a8c98b3-403d-4b9f-810a-8ee27583193a","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:38.388672859Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1269,"session_id":"25d3630b-f888-46e0-9a49-9b647c10c399"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:54:38.388684437Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1269,"session_id":"25d3630b-f888-46e0-9a49-9b647c10c399"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:38.388836375Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1269,"":"io task","client":"1","":"downstairs","session_id":"25d3630b-f888-46e0-9a49-9b647c10c399"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:38.38886795Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1269,"":"io task","client":"0","":"downstairs","session_id":"25d3630b-f888-46e0-9a49-9b647c10c399"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:38.388879329Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1269,"":"io task","client":"2","":"downstairs","session_id":"25d3630b-f888-46e0-9a49-9b647c10c399"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:38.38900244Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1269,"task":"crutest"} {"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:38.389018368Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1269} {"msg":"0a8c98b3-403d-4b9f-810a-8ee27583193a active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:38.389053843Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1269,"session_id":"25d3630b-f888-46e0-9a49-9b647c10c399"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:38.389063411Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1269,"client":"0","":"downstairs","session_id":"25d3630b-f888-46e0-9a49-9b647c10c399"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:38.38907324Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1269,"client":"1","":"downstairs","session_id":"25d3630b-f888-46e0-9a49-9b647c10c399"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:38.389082799Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1269,"client":"2","":"downstairs","session_id":"25d3630b-f888-46e0-9a49-9b647c10c399"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:38.389092557Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1269,"":"io task","client":"2","":"downstairs","session_id":"25d3630b-f888-46e0-9a49-9b647c10c399"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:38.389102406Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1269,"":"io task","client":"0","":"downstairs","session_id":"25d3630b-f888-46e0-9a49-9b647c10c399"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:38.389112414Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1269,"":"io task","client":"1","":"downstairs","session_id":"25d3630b-f888-46e0-9a49-9b647c10c399"} {"msg":"ds_connection connected from Ok(127.0.0.1:41000)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:38.389146479Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1269,"":"io task","client":"2","":"downstairs","session_id":"25d3630b-f888-46e0-9a49-9b647c10c399"} {"msg":"ds_connection connected from Ok(127.0.0.1:46631)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:38.389229137Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1269,"":"io task","client":"1","":"downstairs","session_id":"25d3630b-f888-46e0-9a49-9b647c10c399"} {"msg":"ds_connection connected from Ok(127.0.0.1:42101)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:38.38927583Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1269,"":"io task","client":"0","":"downstairs","session_id":"25d3630b-f888-46e0-9a49-9b647c10c399"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:38.389757399Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1269,"client":"1","":"downstairs","session_id":"25d3630b-f888-46e0-9a49-9b647c10c399"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:38.389855465Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1269,"client":"2","":"downstairs","session_id":"25d3630b-f888-46e0-9a49-9b647c10c399"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:38.389870302Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1269,"client":"0","":"downstairs","session_id":"25d3630b-f888-46e0-9a49-9b647c10c399"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:38.390068553Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1269,"session_id":"25d3630b-f888-46e0-9a49-9b647c10c399"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:38.390129444Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1269,"session_id":"25d3630b-f888-46e0-9a49-9b647c10c399"} {"msg":"[0]R flush_numbers[0..12]: [60, 63, 61, 64, 60, 56, 63, 64, 52, 63, 60, 60]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:38.390285481Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1269,"":"downstairs","":"downstairs","session_id":"25d3630b-f888-46e0-9a49-9b647c10c399"} {"msg":"[0]R generation[0..12]: [30, 32, 31, 33, 30, 28, 32, 33, 26, 32, 30, 30]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:38.390296729Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1269,"":"downstairs","":"downstairs","session_id":"25d3630b-f888-46e0-9a49-9b647c10c399"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:38.390307928Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1269,"":"downstairs","":"downstairs","session_id":"25d3630b-f888-46e0-9a49-9b647c10c399"} {"msg":"[1]R flush_numbers[0..12]: [60, 63, 61, 64, 60, 56, 63, 64, 52, 63, 60, 60]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:38.390318386Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1269,"":"downstairs","":"downstairs","session_id":"25d3630b-f888-46e0-9a49-9b647c10c399"} {"msg":"[1]R generation[0..12]: [30, 32, 31, 33, 30, 28, 32, 33, 26, 32, 30, 30]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:38.390328735Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1269,"":"downstairs","":"downstairs","session_id":"25d3630b-f888-46e0-9a49-9b647c10c399"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:38.390339833Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1269,"":"downstairs","":"downstairs","session_id":"25d3630b-f888-46e0-9a49-9b647c10c399"} {"msg":"[2]R flush_numbers[0..12]: [60, 63, 61, 64, 60, 56, 63, 64, 52, 63, 60, 60]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:38.390350531Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1269,"":"downstairs","":"downstairs","session_id":"25d3630b-f888-46e0-9a49-9b647c10c399"} {"msg":"[2]R generation[0..12]: [30, 32, 31, 33, 30, 28, 32, 33, 26, 32, 30, 30]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:38.39036084Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1269,"":"downstairs","":"downstairs","session_id":"25d3630b-f888-46e0-9a49-9b647c10c399"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:38.390371758Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1269,"":"downstairs","":"downstairs","session_id":"25d3630b-f888-46e0-9a49-9b647c10c399"} {"msg":"Max found gen is 34","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:38.390383027Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1269,"":"downstairs","":"downstairs","session_id":"25d3630b-f888-46e0-9a49-9b647c10c399"} {"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:38.390392765Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1269,"":"downstairs","":"downstairs","session_id":"25d3630b-f888-46e0-9a49-9b647c10c399"} {"msg":"Next flush: 65","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:38.390402214Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1269,"":"downstairs","":"downstairs","session_id":"25d3630b-f888-46e0-9a49-9b647c10c399"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:38.390411602Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1269,"":"downstairs","":"downstairs","session_id":"25d3630b-f888-46e0-9a49-9b647c10c399"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:38.390421611Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1269,"session_id":"25d3630b-f888-46e0-9a49-9b647c10c399"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:38.39043034Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1269,"client":"0","":"downstairs","session_id":"25d3630b-f888-46e0-9a49-9b647c10c399"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:38.390440158Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1269,"client":"1","":"downstairs","session_id":"25d3630b-f888-46e0-9a49-9b647c10c399"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:38.390449787Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1269,"client":"2","":"downstairs","session_id":"25d3630b-f888-46e0-9a49-9b647c10c399"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:38.390459705Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1269,"session_id":"25d3630b-f888-46e0-9a49-9b647c10c399"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:38.390468804Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1269,"session_id":"25d3630b-f888-46e0-9a49-9b647c10c399"} {"msg":"0a8c98b3-403d-4b9f-810a-8ee27583193a is now active with session: 25d3630b-f888-46e0-9a49-9b647c10c399","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:38.390478873Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1269,"session_id":"25d3630b-f888-46e0-9a49-9b647c10c399"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:38.390487731Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1269,"session_id":"25d3630b-f888-46e0-9a49-9b647c10c399"} {"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:38.390497Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1269} {"msg":"Activated sub_volume 0a8c98b3-403d-4b9f-810a-8ee27583193a","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:38.390505399Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1269} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:38.390513497Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1269,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:38.390771969Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1269,"task":"crutest"} 01/30 Write block 477 len 36864 data: 4 5 5 5 5 5 4 4 4 02/30 Read block 565 len 12288 03/30 Read block 5 len 40960 04/30 Read block 492 len 24576 05/30 Write block 290 len 20480 data: 4 4 4 4 3 06/30 Write block 371 len 28672 data: 2 2 2 2 2 2 2 07/30 Write block 12 len 36864 data: 4 4 4 5 6 6 6 6 6 08/30 Read block 496 len 40960 09/30 Read block 470 len 32768 10/30 Write block 159 len 36864 data: 4 3 3 3 3 2 3 3 3 11/30 Write block 447 len 40960 data: 6 6 5 5 6 6 4 3 2 3 12/30 Write block 207 len 8192 data: 7 5 13/30 Read block 321 len 40960 14/30 Read block 150 len 4096 15/30 Read block 289 len 8192 16/30 Read block 53 len 24576 17/30 Write block 502 len 4096 data: 4 18/30 Read block 535 len 4096 19/30 Read block 249 len 40960 20/30 Read block 51 len 24576 21/30 Read block 223 len 20480 22/30 Read block 207 len 8192 23/30 Write block 582 len 32768 data: 4 5 6 6 5 3 2 2 24/30 Read block 465 len 36864 25/30 Read block 116 len 24576 26/30 Write block 264 len 20480 data: 4 4 6 6 3 27/30 Read block 554 len 12288 28/30 Read block 522 len 32768 29/30 Read block 515 len 24576 30/30 Read block 317 len 36864 ---------------------------------------------------------------- Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Read 3 Done Done Done false 1002 Acked Read 10 Done Done Done false 1003 Acked Read 6 Done Done Done false 1004 Acked Write 5 Done Done Done false 1005 Acked Write 7 Done Done Done false 1006 Acked Write 9 Done Done Done false 1007 Acked Read 10 Done Done Done false 1008 Acked Read 8 Sent Done Done false 1009 Acked Write 9 Sent Done Done false 1010 Acked Write 10 Sent Done Done false 1011 Acked Write 2 Sent Done Done false 1012 Acked Read 10 Sent Done Done false 1013 Acked Read 1 Sent Done Done false 1014 Acked Read 2 Sent Done Done false 1015 Acked Read 6 Sent Done Done false 1016 Acked Write 1 Sent Done Done false 1017 Acked Read 1 Sent Done Done false 1018 Acked Read 10 Sent Done Done false 1019 Acked Read 6 Sent Done Done false 1020 Acked Read 5 Sent Done Done false 1021 Acked Read 2 Sent Done Done false 1022 Acked Write 8 Sent Done Done false 1023 Acked Read 9 Sent Done Done false 1024 Acked Read 6 Sent Done Done false 1025 Acked Write 5 Sent Done Done false 1026 Acked Read 3 Sent Done Done false 1027 Acked Read 8 Sent Done Done false 1028 Acked Read 6 Sent Done Done false 1029 Acked Read 9 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 22 0 1 23 Done 8 30 29 67 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:54:38.399954535Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1269} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:38.400134328Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1269,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 30 30 30 60 60 60 T T T 1 020-039 32 32 32 63 63 63 T T T 2 040-059 31 31 31 61 61 61 F F F 3 060-079 33 33 33 64 64 64 F F F 4 080-099 30 30 30 60 60 60 F F F 5 100-119 28 28 28 56 56 56 F F F 6 120-139 32 32 32 63 63 63 F F F 7 140-159 33 33 33 64 64 64 F T T <--- 8 160-179 26 26 26 52 52 52 F T T <--- 9 180-199 32 32 32 63 63 63 F F F 10 200-219 30 30 30 60 60 60 F T T <--- 11 220-239 30 30 30 60 60 60 F F F 12 240-259 32 32 32 63 63 63 F F F 13 260-279 32 32 32 63 63 63 F T T <--- 14 280-299 31 31 31 61 61 61 T T T 15 300-319 32 32 32 63 63 63 F F F 16 320-339 32 32 32 63 63 63 F F F 17 340-359 33 33 33 64 64 64 F F F 18 360-379 32 32 32 63 63 63 T T T 19 380-399 30 30 30 60 60 60 F F F 20 400-419 33 33 33 64 64 64 F F F 21 420-439 25 25 25 51 51 51 F F F 22 440-459 30 30 30 59 59 59 F T T <--- 23 460-479 27 27 27 54 54 54 T T T 24 480-499 28 28 28 55 55 55 T T T 25 500-519 32 32 32 63 63 63 F T T <--- 26 520-539 32 32 32 63 63 63 F F F 27 540-559 32 32 32 63 63 63 F F F 28 560-579 32 32 32 63 63 63 F F F 29 580-599 30 30 30 60 60 60 F T T <--- Max gen: 33, Max flush: 64 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 17 Verifying data now No change after verify Loop: 17 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 65 65 65 F F F 1 020-039 34 34 34 65 65 65 F F F 2 040-059 31 31 31 61 61 61 F F F 3 060-079 33 33 33 64 64 64 F F F 4 080-099 30 30 30 60 60 60 F F F 5 100-119 28 28 28 56 56 56 F F F 6 120-139 32 32 32 63 63 63 F F F 7 140-159 34 34 34 65 65 65 F F F 8 160-179 34 34 34 65 65 65 F F F 9 180-199 32 32 32 63 63 63 F F F 10 200-219 34 34 34 65 65 65 F F F 11 220-239 30 30 30 60 60 60 F F F 12 240-259 32 32 32 63 63 63 F F F 13 260-279 34 34 34 65 65 65 F F F 14 280-299 34 34 34 65 65 65 F F F 15 300-319 32 32 32 63 63 63 F F F 16 320-339 32 32 32 63 63 63 F F F 17 340-359 33 33 33 64 64 64 F F F 18 360-379 34 34 34 65 65 65 F F F 19 380-399 30 30 30 60 60 60 F F F 20 400-419 33 33 33 64 64 64 F F F 21 420-439 25 25 25 51 51 51 F F F 22 440-459 34 34 34 65 65 65 F F F 23 460-479 34 34 34 65 65 65 F F F 24 480-499 34 34 34 65 65 65 F F F 25 500-519 34 34 34 65 65 65 F F F 26 520-539 32 32 32 63 63 63 F F F 27 540-559 32 32 32 63 63 63 F F F 28 560-579 32 32 32 63 63 63 F F F 29 580-599 34 34 34 65 65 65 F F F Max gen: 34, Max flush: 65 Begin loop 18 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:42.658395793Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1277,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:54:42.665432025Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1277,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:42.665488426Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1277,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:42.672233211Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1277,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:42.67251221Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1277} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:42.672541556Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1277} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:42.672550114Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1277} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:42.672561873Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1277} {"msg":"Crucible 369f49a5-70f8-41d9-b1d3-01d9fd0ad1c1 has session id: d84279e1-2688-4f56-92b5-c4cb4eea17f4","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:42.67285111Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1277,"session_id":"d84279e1-2688-4f56-92b5-c4cb4eea17f4"} {"msg":"Upstairs opts: Upstairs UUID: 369f49a5-70f8-41d9-b1d3-01d9fd0ad1c1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:42.672870707Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1277,"session_id":"d84279e1-2688-4f56-92b5-c4cb4eea17f4"} {"msg":"Crucible stats registered with UUID: 369f49a5-70f8-41d9-b1d3-01d9fd0ad1c1","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:42.672880996Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1277,"session_id":"d84279e1-2688-4f56-92b5-c4cb4eea17f4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:54:42.672890124Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1277,"session_id":"d84279e1-2688-4f56-92b5-c4cb4eea17f4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:42.673024994Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1277,"":"io task","client":"2","":"downstairs","session_id":"d84279e1-2688-4f56-92b5-c4cb4eea17f4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:42.673041102Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1277,"":"io task","client":"0","":"downstairs","session_id":"d84279e1-2688-4f56-92b5-c4cb4eea17f4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:42.673051111Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1277,"":"io task","client":"1","":"downstairs","session_id":"d84279e1-2688-4f56-92b5-c4cb4eea17f4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:42.67319255Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1277,"task":"crutest"} {"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:42.673211267Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1277} {"msg":"369f49a5-70f8-41d9-b1d3-01d9fd0ad1c1 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:42.673227435Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1277,"session_id":"d84279e1-2688-4f56-92b5-c4cb4eea17f4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:42.673236303Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1277,"client":"0","":"downstairs","session_id":"d84279e1-2688-4f56-92b5-c4cb4eea17f4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:42.673245992Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1277,"client":"1","":"downstairs","session_id":"d84279e1-2688-4f56-92b5-c4cb4eea17f4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:42.67325567Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1277,"client":"2","":"downstairs","session_id":"d84279e1-2688-4f56-92b5-c4cb4eea17f4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:42.673265249Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1277,"":"io task","client":"2","":"downstairs","session_id":"d84279e1-2688-4f56-92b5-c4cb4eea17f4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:42.673275367Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1277,"":"io task","client":"0","":"downstairs","session_id":"d84279e1-2688-4f56-92b5-c4cb4eea17f4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:42.673285336Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1277,"":"io task","client":"1","":"downstairs","session_id":"d84279e1-2688-4f56-92b5-c4cb4eea17f4"} {"msg":"ds_connection connected from Ok(127.0.0.1:40682)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:42.673366744Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1277,"":"io task","client":"1","":"downstairs","session_id":"d84279e1-2688-4f56-92b5-c4cb4eea17f4"} {"msg":"ds_connection connected from Ok(127.0.0.1:54147)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:42.673446712Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1277,"":"io task","client":"0","":"downstairs","session_id":"d84279e1-2688-4f56-92b5-c4cb4eea17f4"} {"msg":"ds_connection connected from Ok(127.0.0.1:62716)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:42.67346005Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1277,"":"io task","client":"2","":"downstairs","session_id":"d84279e1-2688-4f56-92b5-c4cb4eea17f4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:42.674012559Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1277,"client":"1","":"downstairs","session_id":"d84279e1-2688-4f56-92b5-c4cb4eea17f4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:42.67407334Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1277,"client":"0","":"downstairs","session_id":"d84279e1-2688-4f56-92b5-c4cb4eea17f4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:42.674085148Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1277,"client":"2","":"downstairs","session_id":"d84279e1-2688-4f56-92b5-c4cb4eea17f4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:42.674308575Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1277,"session_id":"d84279e1-2688-4f56-92b5-c4cb4eea17f4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:42.674332162Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1277,"session_id":"d84279e1-2688-4f56-92b5-c4cb4eea17f4"} {"msg":"[0]R flush_numbers[0..12]: [65, 65, 61, 64, 60, 56, 63, 65, 65, 63, 65, 60]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:42.674381994Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1277,"":"downstairs","":"downstairs","session_id":"d84279e1-2688-4f56-92b5-c4cb4eea17f4"} {"msg":"[0]R generation[0..12]: [34, 34, 31, 33, 30, 28, 32, 34, 34, 32, 34, 30]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:42.674393652Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1277,"":"downstairs","":"downstairs","session_id":"d84279e1-2688-4f56-92b5-c4cb4eea17f4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:42.674404701Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1277,"":"downstairs","":"downstairs","session_id":"d84279e1-2688-4f56-92b5-c4cb4eea17f4"} {"msg":"[1]R flush_numbers[0..12]: [65, 65, 61, 64, 60, 56, 63, 65, 65, 63, 65, 60]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:42.674415309Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1277,"":"downstairs","":"downstairs","session_id":"d84279e1-2688-4f56-92b5-c4cb4eea17f4"} {"msg":"[1]R generation[0..12]: [34, 34, 31, 33, 30, 28, 32, 34, 34, 32, 34, 30]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:42.674425948Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1277,"":"downstairs","":"downstairs","session_id":"d84279e1-2688-4f56-92b5-c4cb4eea17f4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:42.674436946Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1277,"":"downstairs","":"downstairs","session_id":"d84279e1-2688-4f56-92b5-c4cb4eea17f4"} {"msg":"[2]R flush_numbers[0..12]: [65, 65, 61, 64, 60, 56, 63, 65, 65, 63, 65, 60]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:42.674447365Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1277,"":"downstairs","":"downstairs","session_id":"d84279e1-2688-4f56-92b5-c4cb4eea17f4"} {"msg":"[2]R generation[0..12]: [34, 34, 31, 33, 30, 28, 32, 34, 34, 32, 34, 30]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:42.674457763Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1277,"":"downstairs","":"downstairs","session_id":"d84279e1-2688-4f56-92b5-c4cb4eea17f4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:42.674468691Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1277,"":"downstairs","":"downstairs","session_id":"d84279e1-2688-4f56-92b5-c4cb4eea17f4"} {"msg":"Max found gen is 35","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:42.67447803Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1277,"":"downstairs","":"downstairs","session_id":"d84279e1-2688-4f56-92b5-c4cb4eea17f4"} {"msg":"Generation requested: 36 >= found:35","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:42.674488009Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1277,"":"downstairs","":"downstairs","session_id":"d84279e1-2688-4f56-92b5-c4cb4eea17f4"} {"msg":"Next flush: 66","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:42.674497567Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1277,"":"downstairs","":"downstairs","session_id":"d84279e1-2688-4f56-92b5-c4cb4eea17f4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:42.674507116Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1277,"":"downstairs","":"downstairs","session_id":"d84279e1-2688-4f56-92b5-c4cb4eea17f4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:42.674516854Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1277,"session_id":"d84279e1-2688-4f56-92b5-c4cb4eea17f4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:42.674527063Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1277,"client":"0","":"downstairs","session_id":"d84279e1-2688-4f56-92b5-c4cb4eea17f4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:42.674536931Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1277,"client":"1","":"downstairs","session_id":"d84279e1-2688-4f56-92b5-c4cb4eea17f4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:42.67454664Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1277,"client":"2","":"downstairs","session_id":"d84279e1-2688-4f56-92b5-c4cb4eea17f4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:42.674556448Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1277,"session_id":"d84279e1-2688-4f56-92b5-c4cb4eea17f4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:42.674565597Z","hostname":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1277,"session_id":"d84279e1-2688-4f56-92b5-c4cb4eea17f4"} {"msg":"369f49a5-70f8-41d9-b1d3-01d9fd0ad1c1 is now active with session: d84279e1-2688-4f56-92b5-c4cb4eea17f4","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:42.674580865Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1277,"session_id":"d84279e1-2688-4f56-92b5-c4cb4eea17f4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:42.674589554Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1277,"session_id":"d84279e1-2688-4f56-92b5-c4cb4eea17f4"} {"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:42.674598522Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1277} {"msg":"Activated sub_volume 369f49a5-70f8-41d9-b1d3-01d9fd0ad1c1","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:42.674606481Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1277} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:42.67461456Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1277,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:42.6748173Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1277,"task":"crutest"} 01/30 Flush 02/30 Write block 48 len 12288 data: 5 4 4 03/30 Read block 553 len 12288 04/30 Read block 325 len 36864 05/30 Read block 585 len 8192 06/30 Read block 258 len 20480 07/30 Read block 244 len 40960 08/30 Write block 583 len 20480 data: 6 7 7 6 4 09/30 Write block 586 len 28672 data: 7 5 3 3 2 5 6 10/30 Write block 528 len 36864 data: 3 3 3 3 3 3 2 2 2 11/30 Write block 320 len 40960 data: 5 4 4 5 5 6 6 5 4 4 12/30 Flush 13/30 Write block 191 len 12288 data: 6 6 6 14/30 Read block 204 len 24576 15/30 Write block 503 len 4096 data: 4 16/30 Write block 269 len 16384 data: 3 3 4 4 17/30 Write block 576 len 28672 data: 4 4 4 4 3 7 5 18/30 Read block 11 len 24576 19/30 Read block 388 len 16384 20/30 Read block 475 len 12288 21/30 Read block 2 len 20480 22/30 Write block 244 len 24576 data: 5 6 6 5 5 4 23/30 Write block 448 len 8192 data: 7 6 24/30 Read block 311 len 40960 25/30 Write block 105 len 16384 data: 3 2 3 3 26/30 Flush 27/30 Write block 272 len 28672 data: 5 3 4 5 5 4 4 28/30 Read block 587 len 20480 29/30 Read block 155 len 16384 30/30 Read block 62 len 12288 ---------------------------------------------------------------- Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 3 Done Done Done false 1002 Acked Read 3 Done Done Done false 1003 Acked Read 9 Sent Done Done false 1004 Acked Read 2 Sent Done Done false 1005 Acked Read 5 Sent Done Done false 1006 Acked Read 10 Sent Done Done false 1007 Acked Write 5 Sent Done Done false 1008 Acked Write 7 Sent Done Done false 1009 Acked Write 9 Sent Done Done false 1010 Acked Write 10 Sent Done Done false 1011 Acked Flush 0 Sent Done Done false 1012 Acked Write 3 Sent Done Done false 1013 Acked Read 6 Sent Done Done false 1014 Acked Write 1 Sent Done Done false 1015 Acked Write 4 Sent Done Done false 1016 Acked Write 7 Sent Done Done false 1017 Acked Read 6 Sent Done Done false 1018 Acked Read 4 Sent Done Done false 1019 Acked Read 3 Sent Done Done false 1020 Acked Read 5 Sent Done Done false 1021 Acked Write 6 Sent Done Done false 1022 Acked Write 2 Sent Done Done false 1023 Acked Read 10 Sent Done Done false 1024 Acked Write 4 Sent Done Done false 1025 Acked Flush 0 Sent Done Done false 1026 Acked Write 7 Sent Done Done false 1027 Acked Read 5 Sent Done Done false 1028 Acked Read 4 Sent Done Done false 1029 Acked Read 3 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 27 0 0 27 Done 2 29 29 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1000)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:54:42.683823211Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1277} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:42.683996006Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1277,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 65 65 65 F F F 1 020-039 34 34 34 65 65 65 F F F 2 040-059 31 36 36 61 67 67 T F F <--- 3 060-079 33 33 33 64 64 64 F F F 4 080-099 30 30 30 60 60 60 F F F 5 100-119 28 36 36 56 68 68 F F F <--- 6 120-139 32 32 32 63 63 63 F F F 7 140-159 34 34 34 65 65 65 F F F 8 160-179 34 34 34 65 65 65 F F F 9 180-199 32 36 36 63 68 68 F F F <--- 10 200-219 34 34 34 65 65 65 F F F 11 220-239 30 30 30 60 60 60 F F F 12 240-259 32 36 36 63 68 68 F F F <--- 13 260-279 34 36 36 65 68 68 F T T <--- 14 280-299 34 34 34 65 65 65 F F F 15 300-319 32 32 32 63 63 63 F F F 16 320-339 32 36 36 63 67 67 F F F <--- 17 340-359 33 33 33 64 64 64 F F F 18 360-379 34 34 34 65 65 65 F F F 19 380-399 30 30 30 60 60 60 F F F 20 400-419 33 33 33 64 64 64 F F F 21 420-439 25 25 25 51 51 51 F F F 22 440-459 34 36 36 65 68 68 F F F <--- 23 460-479 34 34 34 65 65 65 F F F 24 480-499 34 34 34 65 65 65 F F F 25 500-519 34 36 36 65 68 68 F F F <--- 26 520-539 32 36 36 63 67 67 F F F <--- 27 540-559 32 32 32 63 63 63 F F F 28 560-579 32 36 36 63 68 68 F F F <--- 29 580-599 34 36 36 65 68 68 F F F <--- Max gen: 36, Max flush: 68 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 18 Verifying data now No change after verify Loop: 18 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 65 65 65 F F F 1 020-039 34 34 34 65 65 65 F F F 2 040-059 36 36 36 67 67 67 F F F 3 060-079 33 33 33 64 64 64 F F F 4 080-099 30 30 30 60 60 60 F F F 5 100-119 36 36 36 68 68 68 F F F 6 120-139 32 32 32 63 63 63 F F F 7 140-159 34 34 34 65 65 65 F F F 8 160-179 34 34 34 65 65 65 F F F 9 180-199 36 36 36 68 68 68 F F F 10 200-219 34 34 34 65 65 65 F F F 11 220-239 30 30 30 60 60 60 F F F 12 240-259 36 36 36 68 68 68 F F F 13 260-279 37 37 37 69 69 69 F F F 14 280-299 34 34 34 65 65 65 F F F 15 300-319 32 32 32 63 63 63 F F F 16 320-339 36 36 36 67 67 67 F F F 17 340-359 33 33 33 64 64 64 F F F 18 360-379 34 34 34 65 65 65 F F F 19 380-399 30 30 30 60 60 60 F F F 20 400-419 33 33 33 64 64 64 F F F 21 420-439 25 25 25 51 51 51 F F F 22 440-459 36 36 36 68 68 68 F F F 23 460-479 34 34 34 65 65 65 F F F 24 480-499 34 34 34 65 65 65 F F F 25 500-519 36 36 36 68 68 68 F F F 26 520-539 36 36 36 67 67 67 F F F 27 540-559 32 32 32 63 63 63 F F F 28 560-579 36 36 36 68 68 68 F F F 29 580-599 36 36 36 68 68 68 F F F Max gen: 37, Max flush: 69 Begin loop 19 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:46.931393923Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1285,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:46.93960993Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1285,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:46.939903007Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1285} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:46.939928364Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1285} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:46.939936902Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1285} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:46.939948321Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1285} {"msg":"Crucible 55d33542-4d95-4403-8589-76ed67647e6f has session id: 7c4b22a3-52dc-4dc6-9d75-d8f3f37dc43a","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:46.940311737Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1285,"session_id":"7c4b22a3-52dc-4dc6-9d75-d8f3f37dc43a"} {"msg":"Upstairs opts: Upstairs UUID: 55d33542-4d95-4403-8589-76ed67647e6f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:46.940340863Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1285,"session_id":"7c4b22a3-52dc-4dc6-9d75-d8f3f37dc43a"} {"msg":"Crucible stats registered with UUID: 55d33542-4d95-4403-8589-76ed67647e6f","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:46.940351911Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1285,"session_id":"7c4b22a3-52dc-4dc6-9d75-d8f3f37dc43a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:54:46.94036111Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1285,"session_id":"7c4b22a3-52dc-4dc6-9d75-d8f3f37dc43a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:46.940479532Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1285,"":"io task","client":"1","":"downstairs","session_id":"7c4b22a3-52dc-4dc6-9d75-d8f3f37dc43a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:46.94049608Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1285,"":"io task","client":"0","":"downstairs","session_id":"7c4b22a3-52dc-4dc6-9d75-d8f3f37dc43a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:46.940506208Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1285,"":"io task","client":"2","":"downstairs","session_id":"7c4b22a3-52dc-4dc6-9d75-d8f3f37dc43a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:46.940659466Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1285,"task":"crutest"} {"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:46.940677483Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1285} {"msg":"55d33542-4d95-4403-8589-76ed67647e6f active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:46.940704529Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1285,"session_id":"7c4b22a3-52dc-4dc6-9d75-d8f3f37dc43a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:46.940714118Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1285,"client":"0","":"downstairs","session_id":"7c4b22a3-52dc-4dc6-9d75-d8f3f37dc43a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:46.940723886Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1285,"client":"1","":"downstairs","session_id":"7c4b22a3-52dc-4dc6-9d75-d8f3f37dc43a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:46.940733635Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1285,"client":"2","":"downstairs","session_id":"7c4b22a3-52dc-4dc6-9d75-d8f3f37dc43a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:46.940743323Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1285,"":"io task","client":"2","":"downstairs","session_id":"7c4b22a3-52dc-4dc6-9d75-d8f3f37dc43a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:46.940753272Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1285,"":"io task","client":"0","":"downstairs","session_id":"7c4b22a3-52dc-4dc6-9d75-d8f3f37dc43a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:46.94076353Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1285,"":"io task","client":"1","":"downstairs","session_id":"7c4b22a3-52dc-4dc6-9d75-d8f3f37dc43a"} {"msg":"ds_connection connected from Ok(127.0.0.1:46056)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:46.940840329Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1285,"":"io task","client":"1","":"downstairs","session_id":"7c4b22a3-52dc-4dc6-9d75-d8f3f37dc43a"} {"msg":"ds_connection connected from Ok(127.0.0.1:57384)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:46.940943964Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1285,"":"io task","client":"0","":"downstairs","session_id":"7c4b22a3-52dc-4dc6-9d75-d8f3f37dc43a"} {"msg":"ds_connection connected from Ok(127.0.0.1:61384)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:46.94096619Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1285,"":"io task","client":"2","":"downstairs","session_id":"7c4b22a3-52dc-4dc6-9d75-d8f3f37dc43a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:46.941457168Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1285,"client":"1","":"downstairs","session_id":"7c4b22a3-52dc-4dc6-9d75-d8f3f37dc43a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:46.941474595Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1285,"client":"0","":"downstairs","session_id":"7c4b22a3-52dc-4dc6-9d75-d8f3f37dc43a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:46.94157661Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1285,"client":"2","":"downstairs","session_id":"7c4b22a3-52dc-4dc6-9d75-d8f3f37dc43a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:46.94178142Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1285,"session_id":"7c4b22a3-52dc-4dc6-9d75-d8f3f37dc43a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:46.941799817Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1285,"session_id":"7c4b22a3-52dc-4dc6-9d75-d8f3f37dc43a"} {"msg":"[0]R flush_numbers[0..12]: [65, 65, 67, 64, 60, 68, 63, 65, 65, 68, 65, 60]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:46.941875426Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1285,"":"downstairs","":"downstairs","session_id":"7c4b22a3-52dc-4dc6-9d75-d8f3f37dc43a"} {"msg":"[0]R generation[0..12]: [34, 34, 36, 33, 30, 36, 32, 34, 34, 36, 34, 30]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:46.941886655Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1285,"":"downstairs","":"downstairs","session_id":"7c4b22a3-52dc-4dc6-9d75-d8f3f37dc43a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:46.941897843Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1285,"":"downstairs","":"downstairs","session_id":"7c4b22a3-52dc-4dc6-9d75-d8f3f37dc43a"} {"msg":"[1]R flush_numbers[0..12]: [65, 65, 67, 64, 60, 68, 63, 65, 65, 68, 65, 60]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:46.941908231Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1285,"":"downstairs","":"downstairs","session_id":"7c4b22a3-52dc-4dc6-9d75-d8f3f37dc43a"} {"msg":"[1]R generation[0..12]: [34, 34, 36, 33, 30, 36, 32, 34, 34, 36, 34, 30]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:46.94191848Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1285,"":"downstairs","":"downstairs","session_id":"7c4b22a3-52dc-4dc6-9d75-d8f3f37dc43a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:46.941929438Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1285,"":"downstairs","":"downstairs","session_id":"7c4b22a3-52dc-4dc6-9d75-d8f3f37dc43a"} {"msg":"[2]R flush_numbers[0..12]: [65, 65, 67, 64, 60, 68, 63, 65, 65, 68, 65, 60]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:46.941942136Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1285,"":"downstairs","":"downstairs","session_id":"7c4b22a3-52dc-4dc6-9d75-d8f3f37dc43a"} {"msg":"[2]R generation[0..12]: [34, 34, 36, 33, 30, 36, 32, 34, 34, 36, 34, 30]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:46.941952505Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1285,"":"downstairs","":"downstairs","session_id":"7c4b22a3-52dc-4dc6-9d75-d8f3f37dc43a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:46.941963353Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1285,"":"downstairs","":"downstairs","session_id":"7c4b22a3-52dc-4dc6-9d75-d8f3f37dc43a"} {"msg":"Max found gen is 38","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:46.941972872Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1285,"":"downstairs","":"downstairs","session_id":"7c4b22a3-52dc-4dc6-9d75-d8f3f37dc43a"} {"msg":"Generation requested: 38 >= found:38","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:46.94198255Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1285,"":"downstairs","":"downstairs","session_id":"7c4b22a3-52dc-4dc6-9d75-d8f3f37dc43a"} {"msg":"Next flush: 70","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:46.941991849Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1285,"":"downstairs","":"downstairs","session_id":"7c4b22a3-52dc-4dc6-9d75-d8f3f37dc43a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:46.942001468Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1285,"":"downstairs","":"downstairs","session_id":"7c4b22a3-52dc-4dc6-9d75-d8f3f37dc43a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:46.942011216Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1285,"session_id":"7c4b22a3-52dc-4dc6-9d75-d8f3f37dc43a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:46.942019785Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1285,"client":"0","":"downstairs","session_id":"7c4b22a3-52dc-4dc6-9d75-d8f3f37dc43a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:46.942029404Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1285,"client":"1","":"downstairs","session_id":"7c4b22a3-52dc-4dc6-9d75-d8f3f37dc43a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:46.942039152Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1285,"client":"2","":"downstairs","session_id":"7c4b22a3-52dc-4dc6-9d75-d8f3f37dc43a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:46.942048991Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1285,"session_id":"7c4b22a3-52dc-4dc6-9d75-d8f3f37dc43a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:46.942058129Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1285,"session_id":"7c4b22a3-52dc-4dc6-9d75-d8f3f37dc43a"} {"msg":"55d33542-4d95-4403-8589-76ed67647e6f is now active with session: 7c4b22a3-52dc-4dc6-9d75-d8f3f37dc43a","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:46.942068208Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1285,"session_id":"7c4b22a3-52dc-4dc6-9d75-d8f3f37dc43a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:46.942076957Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1285,"session_id":"7c4b22a3-52dc-4dc6-9d75-d8f3f37dc43a"} {"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:46.942087815Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1285} {"msg":"Activated sub_volume 55d33542-4d95-4403-8589-76ed67647e6f","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:46.942096014Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1285} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:46.942104172Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1285,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:46.942385361Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1285,"task":"crutest"} 01/30 Write block 587 len 36864 data: 6 4 4 3 6 7 7 6 5 02/30 Write block 398 len 4096 data: 2 03/30 Write block 425 len 32768 data: 4 3 4 4 4 3 4 5 04/30 Write block 305 len 8192 data: 3 3 05/30 Write block 545 len 36864 data: 5 4 2 2 2 3 3 5 5 06/30 Flush 07/30 Write block 37 len 8192 data: 6 8 08/30 Write block 310 len 4096 data: 5 09/30 Write block 490 len 12288 data: 4 3 3 10/30 Read block 88 len 4096 11/30 Write block 240 len 32768 data: 5 5 5 5 6 7 7 6 12/30 Write block 261 len 24576 data: 5 4 6 5 5 7 13/30 Write block 405 len 28672 data: 3 4 4 3 3 6 8 14/30 Write block 191 len 36864 data: 7 7 7 6 4 4 5 5 4 15/30 Read block 289 len 12288 16/30 Write block 515 len 28672 data: 6 4 5 7 6 5 5 17/30 Flush 18/30 Write block 346 len 32768 data: 3 3 3 2 3 4 4 4 19/30 Read block 389 len 16384 20/30 Read block 417 len 36864 21/30 Write block 233 len 16384 data: 2 2 2 3 22/30 Read block 381 len 36864 23/30 Read block 250 len 24576 24/30 Read block 145 len 12288 25/30 Read block 459 len 20480 26/30 Read block 335 len 12288 27/30 Read block 108 len 4096 28/30 Write block 437 len 16384 data: 4 5 4 2 29/30 Write block 453 len 32768 data: 5 4 3 4 4 4 4 2 30/30 Read block 127 len 24576 ---------------------------------------------------------------- Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 2 Done Done Sent false 1007 Acked Write 1 Done Done Sent false 1008 Acked Write 3 Done Done Sent false 1009 Acked Read 1 Done Done Sent false 1010 Acked Write 8 Done Done Sent false 1011 Acked Write 6 Done Done Sent false 1012 Acked Write 7 Done Done Sent false 1013 Acked Write 9 Done Done Sent false 1014 Acked Read 3 Done Done Sent false 1015 Acked Write 7 Done Done Sent false 1016 Acked Flush 0 Done Done Sent false 1017 Acked Write 8 Done Done Sent false 1018 Acked Read 4 Done Done Sent false 1019 Acked Read 9 Done Done Sent false 1020 Acked Write 4 Done Done Sent false 1021 Acked Read 9 Done Done Sent false 1022 Acked Read 6 Done Done Sent false 1023 Acked Read 3 Done Done Sent false 1024 Acked Read 5 Done Done Sent false 1025 Acked Read 3 Done Done Sent false 1026 Acked Read 1 Done Done Sent false 1027 Acked Write 4 Done Done Sent false 1028 Acked Write 8 Done Done Sent false 1029 Acked Read 6 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 24 25 Done 23 24 0 47 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) Some(JobId(1005)) Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:54:46.950182941Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1285} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:46.950358035Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1285,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 65 65 65 F F F 1 020-039 38 38 34 71 71 65 F F F <--- 2 040-059 36 36 36 67 67 67 F F F 3 060-079 33 33 33 64 64 64 F F F 4 080-099 30 30 30 60 60 60 F F F 5 100-119 36 36 36 68 68 68 F F F 6 120-139 32 32 32 63 63 63 F F F 7 140-159 34 34 34 65 65 65 F F F 8 160-179 34 34 34 65 65 65 F F F 9 180-199 38 38 36 71 71 68 F F F <--- 10 200-219 34 34 34 65 65 65 F F F 11 220-239 30 30 30 60 60 60 T T F <--- 12 240-259 38 38 36 71 71 68 F F F <--- 13 260-279 38 38 37 71 71 69 F F F <--- 14 280-299 34 34 34 65 65 65 F F F 15 300-319 38 38 38 71 71 70 F F F <--- 16 320-339 36 36 36 67 67 67 F F F 17 340-359 33 33 33 64 64 64 T T F <--- 18 360-379 34 34 34 65 65 65 F F F 19 380-399 38 38 38 70 70 70 F F F 20 400-419 38 38 33 71 71 64 F F F <--- 21 420-439 38 38 38 70 70 70 T T F <--- 22 440-459 36 36 36 68 68 68 T T F <--- 23 460-479 34 34 34 65 65 65 T T F <--- 24 480-499 38 38 34 71 71 65 F F F <--- 25 500-519 38 38 36 71 71 68 F F F <--- 26 520-539 38 38 36 71 71 67 F F F <--- 27 540-559 38 38 38 70 70 70 F F F 28 560-579 36 36 36 68 68 68 F F F 29 580-599 38 38 38 70 70 70 F F F Max gen: 38, Max flush: 71 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 19 Verifying data now No change after verify Loop: 19 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 65 65 65 F F F 1 020-039 38 38 38 71 71 71 F F F 2 040-059 36 36 36 67 67 67 F F F 3 060-079 33 33 33 64 64 64 F F F 4 080-099 30 30 30 60 60 60 F F F 5 100-119 36 36 36 68 68 68 F F F 6 120-139 32 32 32 63 63 63 F F F 7 140-159 34 34 34 65 65 65 F F F 8 160-179 34 34 34 65 65 65 F F F 9 180-199 38 38 38 71 71 71 F F F 10 200-219 34 34 34 65 65 65 F F F 11 220-239 39 39 39 72 72 72 F F F 12 240-259 38 38 38 71 71 71 F F F 13 260-279 38 38 38 71 71 71 F F F 14 280-299 34 34 34 65 65 65 F F F 15 300-319 38 38 38 71 71 71 F F F 16 320-339 36 36 36 67 67 67 F F F 17 340-359 39 39 39 72 72 72 F F F 18 360-379 34 34 34 65 65 65 F F F 19 380-399 38 38 38 70 70 70 F F F 20 400-419 38 38 38 71 71 71 F F F 21 420-439 39 39 39 72 72 72 F F F 22 440-459 39 39 39 72 72 72 F F F 23 460-479 39 39 39 72 72 72 F F F 24 480-499 38 38 38 71 71 71 F F F 25 500-519 38 38 38 71 71 71 F F F 26 520-539 38 38 38 71 71 71 F F F 27 540-559 38 38 38 70 70 70 F F F 28 560-579 36 36 36 68 68 68 F F F 29 580-599 38 38 38 70 70 70 F F F Max gen: 39, Max flush: 72 Begin loop 20 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:51.199554607Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1293,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:54:51.206633263Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1293,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:51.206685785Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1293,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:51.213459426Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1293,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:51.213724176Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1293} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:51.213753032Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1293} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:51.213761531Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1293} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:51.213773109Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1293} {"msg":"Crucible 3462bbd3-9fa9-462d-b891-693716cdb7f6 has session id: afaf4e74-a3e4-4649-9237-cb91f45f6425","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:51.214116019Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1293,"session_id":"afaf4e74-a3e4-4649-9237-cb91f45f6425"} {"msg":"Upstairs opts: Upstairs UUID: 3462bbd3-9fa9-462d-b891-693716cdb7f6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:51.214146534Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1293,"session_id":"afaf4e74-a3e4-4649-9237-cb91f45f6425"} {"msg":"Crucible stats registered with UUID: 3462bbd3-9fa9-462d-b891-693716cdb7f6","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:51.214157842Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1293,"session_id":"afaf4e74-a3e4-4649-9237-cb91f45f6425"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:54:51.214167021Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1293,"session_id":"afaf4e74-a3e4-4649-9237-cb91f45f6425"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:51.214294322Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1293,"":"io task","client":"1","":"downstairs","session_id":"afaf4e74-a3e4-4649-9237-cb91f45f6425"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:51.21431046Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1293,"":"io task","client":"0","":"downstairs","session_id":"afaf4e74-a3e4-4649-9237-cb91f45f6425"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:51.214320668Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1293,"":"io task","client":"2","":"downstairs","session_id":"afaf4e74-a3e4-4649-9237-cb91f45f6425"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:51.214462268Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1293,"task":"crutest"} {"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:51.214485354Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1293} {"msg":"3462bbd3-9fa9-462d-b891-693716cdb7f6 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:51.214494513Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1293,"session_id":"afaf4e74-a3e4-4649-9237-cb91f45f6425"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:51.214503581Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1293,"client":"0","":"downstairs","session_id":"afaf4e74-a3e4-4649-9237-cb91f45f6425"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:51.21451367Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1293,"client":"1","":"downstairs","session_id":"afaf4e74-a3e4-4649-9237-cb91f45f6425"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:51.214523489Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1293,"client":"2","":"downstairs","session_id":"afaf4e74-a3e4-4649-9237-cb91f45f6425"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:51.214533267Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1293,"":"io task","client":"2","":"downstairs","session_id":"afaf4e74-a3e4-4649-9237-cb91f45f6425"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:51.214543566Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1293,"":"io task","client":"0","":"downstairs","session_id":"afaf4e74-a3e4-4649-9237-cb91f45f6425"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:51.214553984Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1293,"":"io task","client":"1","":"downstairs","session_id":"afaf4e74-a3e4-4649-9237-cb91f45f6425"} {"msg":"ds_connection connected from Ok(127.0.0.1:41499)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:51.214576841Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1293,"":"io task","client":"0","":"downstairs","session_id":"afaf4e74-a3e4-4649-9237-cb91f45f6425"} {"msg":"ds_connection connected from Ok(127.0.0.1:53413)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:51.214595368Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1293,"":"io task","client":"1","":"downstairs","session_id":"afaf4e74-a3e4-4649-9237-cb91f45f6425"} {"msg":"ds_connection connected from Ok(127.0.0.1:46893)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:51.21464787Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1293,"":"io task","client":"2","":"downstairs","session_id":"afaf4e74-a3e4-4649-9237-cb91f45f6425"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:51.21525603Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1293,"client":"1","":"downstairs","session_id":"afaf4e74-a3e4-4649-9237-cb91f45f6425"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:51.215343198Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1293,"client":"0","":"downstairs","session_id":"afaf4e74-a3e4-4649-9237-cb91f45f6425"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:51.215364414Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1293,"client":"2","":"downstairs","session_id":"afaf4e74-a3e4-4649-9237-cb91f45f6425"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:51.21559774Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1293,"session_id":"afaf4e74-a3e4-4649-9237-cb91f45f6425"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:51.215622506Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1293,"session_id":"afaf4e74-a3e4-4649-9237-cb91f45f6425"} {"msg":"[0]R flush_numbers[0..12]: [65, 71, 67, 64, 60, 68, 63, 65, 65, 71, 65, 72]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:51.215661241Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1293,"":"downstairs","":"downstairs","session_id":"afaf4e74-a3e4-4649-9237-cb91f45f6425"} {"msg":"[0]R generation[0..12]: [34, 38, 36, 33, 30, 36, 32, 34, 34, 38, 34, 39]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:51.215672349Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1293,"":"downstairs","":"downstairs","session_id":"afaf4e74-a3e4-4649-9237-cb91f45f6425"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:51.215683437Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1293,"":"downstairs","":"downstairs","session_id":"afaf4e74-a3e4-4649-9237-cb91f45f6425"} {"msg":"[1]R flush_numbers[0..12]: [65, 71, 67, 64, 60, 68, 63, 65, 65, 71, 65, 72]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:51.215694016Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1293,"":"downstairs","":"downstairs","session_id":"afaf4e74-a3e4-4649-9237-cb91f45f6425"} {"msg":"[1]R generation[0..12]: [34, 38, 36, 33, 30, 36, 32, 34, 34, 38, 34, 39]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:51.215704464Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1293,"":"downstairs","":"downstairs","session_id":"afaf4e74-a3e4-4649-9237-cb91f45f6425"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:51.215715443Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1293,"":"downstairs","":"downstairs","session_id":"afaf4e74-a3e4-4649-9237-cb91f45f6425"} {"msg":"[2]R flush_numbers[0..12]: [65, 71, 67, 64, 60, 68, 63, 65, 65, 71, 65, 72]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:51.215725801Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1293,"":"downstairs","":"downstairs","session_id":"afaf4e74-a3e4-4649-9237-cb91f45f6425"} {"msg":"[2]R generation[0..12]: [34, 38, 36, 33, 30, 36, 32, 34, 34, 38, 34, 39]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:51.21573617Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1293,"":"downstairs","":"downstairs","session_id":"afaf4e74-a3e4-4649-9237-cb91f45f6425"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:51.215747058Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1293,"":"downstairs","":"downstairs","session_id":"afaf4e74-a3e4-4649-9237-cb91f45f6425"} {"msg":"Max found gen is 40","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:51.215756747Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1293,"":"downstairs","":"downstairs","session_id":"afaf4e74-a3e4-4649-9237-cb91f45f6425"} {"msg":"Generation requested: 40 >= found:40","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:51.215768135Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1293,"":"downstairs","":"downstairs","session_id":"afaf4e74-a3e4-4649-9237-cb91f45f6425"} {"msg":"Next flush: 73","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:51.215777394Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1293,"":"downstairs","":"downstairs","session_id":"afaf4e74-a3e4-4649-9237-cb91f45f6425"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:51.215786882Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1293,"":"downstairs","":"downstairs","session_id":"afaf4e74-a3e4-4649-9237-cb91f45f6425"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:51.215796801Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1293,"session_id":"afaf4e74-a3e4-4649-9237-cb91f45f6425"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:51.215805869Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1293,"client":"0","":"downstairs","session_id":"afaf4e74-a3e4-4649-9237-cb91f45f6425"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:51.215815728Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1293,"client":"1","":"downstairs","session_id":"afaf4e74-a3e4-4649-9237-cb91f45f6425"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:51.215825666Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1293,"client":"2","":"downstairs","session_id":"afaf4e74-a3e4-4649-9237-cb91f45f6425"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:51.215835585Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1293,"session_id":"afaf4e74-a3e4-4649-9237-cb91f45f6425"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:51.215844774Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1293,"session_id":"afaf4e74-a3e4-4649-9237-cb91f45f6425"} {"msg":"3462bbd3-9fa9-462d-b891-693716cdb7f6 is now active with session: afaf4e74-a3e4-4649-9237-cb91f45f6425","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:51.215854682Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1293,"session_id":"afaf4e74-a3e4-4649-9237-cb91f45f6425"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:51.215863361Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1293,"session_id":"afaf4e74-a3e4-4649-9237-cb91f45f6425"} {"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:51.215872469Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1293} {"msg":"Activated sub_volume 3462bbd3-9fa9-462d-b891-693716cdb7f6","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:51.215880358Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1293} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:51.215888617Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1293,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:51.216153928Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1293,"task":"crutest"} 01/30 Write block 341 len 4096 data: 3 02/30 Flush 03/30 Write block 55 len 8192 data: 4 4 04/30 Write block 468 len 36864 data: 5 5 5 6 5 3 4 4 5 05/30 Read block 200 len 4096 06/30 Read block 273 len 4096 07/30 Write block 474 len 28672 data: 5 5 6 5 6 6 6 08/30 Flush 09/30 Write block 564 len 8192 data: 5 4 10/30 Read block 157 len 32768 11/30 Read block 78 len 20480 12/30 Read block 509 len 4096 13/30 Write block 187 len 36864 data: 2 2 3 4 8 8 8 7 5 14/30 Read block 325 len 36864 15/30 Read block 294 len 40960 16/30 Write block 70 len 40960 data: 4 4 4 4 4 3 3 3 4 3 17/30 Write block 510 len 20480 data: 5 4 5 6 5 18/30 Write block 532 len 16384 data: 4 4 3 3 19/30 Read block 244 len 12288 20/30 Write block 364 len 36864 data: 6 6 5 3 3 3 2 3 3 21/30 Flush 22/30 Write block 264 len 40960 data: 6 6 8 7 4 4 4 5 6 4 23/30 Write block 537 len 28672 data: 2 3 3 3 3 3 4 24/30 Read block 327 len 16384 25/30 Write block 228 len 24576 data: 3 2 2 2 2 3 26/30 Write block 519 len 16384 data: 7 6 6 5 27/30 Read block 187 len 8192 28/30 Write block 275 len 36864 data: 6 6 5 5 3 3 2 2 3 29/30 Write block 494 len 40960 data: 3 3 3 3 2 3 3 3 5 5 30/30 Write block 29 len 12288 data: 5 4 5 ---------------------------------------------------------------- Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Flush 0 Sent Done Done false 1002 Acked Write 2 Sent Done Done false 1003 Acked Write 9 Sent Done Done false 1004 Acked Read 1 Sent Done Done false 1005 Acked Read 1 Sent Done Done false 1006 Acked Write 7 Sent Done Done false 1007 Acked Flush 0 Sent Done Done false 1008 Acked Write 2 Sent Done Done false 1009 Acked Read 8 Sent Done Done false 1010 Acked Read 5 Sent Done Done false 1011 Acked Read 1 Sent Done Done false 1012 Acked Write 9 Sent Done Done false 1013 Acked Read 9 Sent Done Done false 1014 Acked Read 10 Sent Done Done false 1015 Acked Write 10 Sent Done Done false 1016 Acked Write 5 Sent Done Done false 1017 Acked Write 4 Sent Done Done false 1018 Acked Read 3 Sent Done Done false 1019 Acked Write 9 Sent Done Done false 1020 Acked Flush 0 Sent Done Done false 1021 Acked Write 10 Sent Done Done false 1022 Acked Write 7 Sent Done Done false 1023 Acked Read 4 Sent Done Done false 1024 Acked Write 6 Sent Done Done false 1025 Acked Write 4 Sent Done Done false 1026 Acked Read 2 Sent Done Done false 1027 Acked Write 9 Sent Sent Sent false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 3 3 35 Done 1 27 27 55 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:54:51.224791164Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1293} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:51.224978046Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1293,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 65 65 65 F F F 1 020-039 38 38 38 71 71 71 F T T <--- 2 040-059 36 40 40 67 74 74 F F F <--- 3 060-079 33 40 40 64 75 75 F F F <--- 4 080-099 30 30 30 60 60 60 F F F 5 100-119 36 36 36 68 68 68 F F F 6 120-139 32 32 32 63 63 63 F F F 7 140-159 34 34 34 65 65 65 F F F 8 160-179 34 34 34 65 65 65 F F F 9 180-199 38 40 40 71 75 75 F F F <--- 10 200-219 34 34 34 65 65 65 F F F 11 220-239 39 39 39 72 72 72 F T T <--- 12 240-259 38 38 38 71 71 71 F F F 13 260-279 38 38 38 71 71 71 F T T <--- 14 280-299 34 34 34 65 65 65 F T T <--- 15 300-319 38 38 38 71 71 71 F F F 16 320-339 36 36 36 67 67 67 F F F 17 340-359 39 40 40 72 73 73 T F F <--- 18 360-379 34 40 40 65 75 75 F F F <--- 19 380-399 38 38 38 70 70 70 F F F 20 400-419 38 38 38 71 71 71 F F F 21 420-439 39 39 39 72 72 72 F F F 22 440-459 39 39 39 72 72 72 F F F 23 460-479 39 40 40 72 74 74 F F F <--- 24 480-499 38 40 40 71 74 74 F T T <--- 25 500-519 38 40 40 71 75 75 F T T <--- 26 520-539 38 40 40 71 75 75 F T T <--- 27 540-559 38 38 38 70 70 70 F T T <--- 28 560-579 36 40 40 68 75 75 F F F <--- 29 580-599 38 38 38 70 70 70 F F F Max gen: 40, Max flush: 75 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 20 Verifying data now No change after verify Loop: 20 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 65 65 65 F F F 1 020-039 41 41 41 76 76 76 F F F 2 040-059 40 40 40 74 74 74 F F F 3 060-079 40 40 40 75 75 75 F F F 4 080-099 30 30 30 60 60 60 F F F 5 100-119 36 36 36 68 68 68 F F F 6 120-139 32 32 32 63 63 63 F F F 7 140-159 34 34 34 65 65 65 F F F 8 160-179 34 34 34 65 65 65 F F F 9 180-199 40 40 40 75 75 75 F F F 10 200-219 34 34 34 65 65 65 F F F 11 220-239 41 41 41 76 76 76 F F F 12 240-259 38 38 38 71 71 71 F F F 13 260-279 41 41 41 76 76 76 F F F 14 280-299 41 41 41 76 76 76 F F F 15 300-319 38 38 38 71 71 71 F F F 16 320-339 36 36 36 67 67 67 F F F 17 340-359 40 40 40 73 73 73 F F F 18 360-379 40 40 40 75 75 75 F F F 19 380-399 38 38 38 70 70 70 F F F 20 400-419 38 38 38 71 71 71 F F F 21 420-439 39 39 39 72 72 72 F F F 22 440-459 39 39 39 72 72 72 F F F 23 460-479 40 40 40 74 74 74 F F F 24 480-499 41 41 41 76 76 76 F F F 25 500-519 41 41 41 76 76 76 F F F 26 520-539 41 41 41 76 76 76 F F F 27 540-559 41 41 41 76 76 76 F F F 28 560-579 40 40 40 75 75 75 F F F 29 580-599 38 38 38 70 70 70 F F F Max gen: 41, Max flush: 76 Begin loop 21 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:55.483623434Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1301,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:55.49185439Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1301,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:55.492152916Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1301} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:55.492177802Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1301} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:55.492186191Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1301} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:55.492197619Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1301} {"msg":"Crucible 0f4da21a-8eba-466b-8c42-984b9fa8f4d1 has session id: cc08105e-b5db-490d-a8d2-64e036755fce","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:55.492522201Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1301,"session_id":"cc08105e-b5db-490d-a8d2-64e036755fce"} {"msg":"Upstairs opts: Upstairs UUID: 0f4da21a-8eba-466b-8c42-984b9fa8f4d1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:55.492544398Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1301,"session_id":"cc08105e-b5db-490d-a8d2-64e036755fce"} {"msg":"Crucible stats registered with UUID: 0f4da21a-8eba-466b-8c42-984b9fa8f4d1","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:55.492554427Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1301,"session_id":"cc08105e-b5db-490d-a8d2-64e036755fce"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:54:55.492563485Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1301,"session_id":"cc08105e-b5db-490d-a8d2-64e036755fce"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:55.492723332Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1301,"":"io task","client":"1","":"downstairs","session_id":"cc08105e-b5db-490d-a8d2-64e036755fce"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:55.492753197Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1301,"":"io task","client":"0","":"downstairs","session_id":"cc08105e-b5db-490d-a8d2-64e036755fce"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:55.492764036Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1301,"":"io task","client":"2","":"downstairs","session_id":"cc08105e-b5db-490d-a8d2-64e036755fce"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:55.492890547Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1301,"task":"crutest"} {"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:55.492907524Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1301} {"msg":"0f4da21a-8eba-466b-8c42-984b9fa8f4d1 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:55.49294076Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1301,"session_id":"cc08105e-b5db-490d-a8d2-64e036755fce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:55.492950378Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1301,"client":"0","":"downstairs","session_id":"cc08105e-b5db-490d-a8d2-64e036755fce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:55.492962246Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1301,"client":"1","":"downstairs","session_id":"cc08105e-b5db-490d-a8d2-64e036755fce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:55.492972225Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1301,"client":"2","":"downstairs","session_id":"cc08105e-b5db-490d-a8d2-64e036755fce"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:55.492981884Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1301,"":"io task","client":"2","":"downstairs","session_id":"cc08105e-b5db-490d-a8d2-64e036755fce"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:55.492991962Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1301,"":"io task","client":"0","":"downstairs","session_id":"cc08105e-b5db-490d-a8d2-64e036755fce"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:55.493001721Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1301,"":"io task","client":"1","":"downstairs","session_id":"cc08105e-b5db-490d-a8d2-64e036755fce"} {"msg":"ds_connection connected from Ok(127.0.0.1:51780)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:55.493095457Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1301,"":"io task","client":"0","":"downstairs","session_id":"cc08105e-b5db-490d-a8d2-64e036755fce"} {"msg":"ds_connection connected from Ok(127.0.0.1:48089)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:55.493116204Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1301,"":"io task","client":"1","":"downstairs","session_id":"cc08105e-b5db-490d-a8d2-64e036755fce"} {"msg":"ds_connection connected from Ok(127.0.0.1:55663)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:55.493126472Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1301,"":"io task","client":"2","":"downstairs","session_id":"cc08105e-b5db-490d-a8d2-64e036755fce"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:55.493778916Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1301,"client":"0","":"downstairs","session_id":"cc08105e-b5db-490d-a8d2-64e036755fce"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:55.493797923Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1301,"client":"1","":"downstairs","session_id":"cc08105e-b5db-490d-a8d2-64e036755fce"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:55.493808941Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1301,"client":"2","":"downstairs","session_id":"cc08105e-b5db-490d-a8d2-64e036755fce"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:55.494134084Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1301,"session_id":"cc08105e-b5db-490d-a8d2-64e036755fce"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:55.494143972Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1301,"session_id":"cc08105e-b5db-490d-a8d2-64e036755fce"} {"msg":"[0]R flush_numbers[0..12]: [65, 76, 74, 75, 60, 68, 63, 65, 65, 75, 65, 76]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:55.494163239Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1301,"":"downstairs","":"downstairs","session_id":"cc08105e-b5db-490d-a8d2-64e036755fce"} {"msg":"[0]R generation[0..12]: [34, 41, 40, 40, 30, 36, 32, 34, 34, 40, 34, 41]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:55.494173778Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1301,"":"downstairs","":"downstairs","session_id":"cc08105e-b5db-490d-a8d2-64e036755fce"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:55.494187486Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1301,"":"downstairs","":"downstairs","session_id":"cc08105e-b5db-490d-a8d2-64e036755fce"} {"msg":"[1]R flush_numbers[0..12]: [65, 76, 74, 75, 60, 68, 63, 65, 65, 75, 65, 76]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:55.494197934Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1301,"":"downstairs","":"downstairs","session_id":"cc08105e-b5db-490d-a8d2-64e036755fce"} {"msg":"[1]R generation[0..12]: [34, 41, 40, 40, 30, 36, 32, 34, 34, 40, 34, 41]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:55.494208473Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1301,"":"downstairs","":"downstairs","session_id":"cc08105e-b5db-490d-a8d2-64e036755fce"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:55.494219261Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1301,"":"downstairs","":"downstairs","session_id":"cc08105e-b5db-490d-a8d2-64e036755fce"} {"msg":"[2]R flush_numbers[0..12]: [65, 76, 74, 75, 60, 68, 63, 65, 65, 75, 65, 76]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:55.494229829Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1301,"":"downstairs","":"downstairs","session_id":"cc08105e-b5db-490d-a8d2-64e036755fce"} {"msg":"[2]R generation[0..12]: [34, 41, 40, 40, 30, 36, 32, 34, 34, 40, 34, 41]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:55.494240238Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1301,"":"downstairs","":"downstairs","session_id":"cc08105e-b5db-490d-a8d2-64e036755fce"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:55.494251176Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1301,"":"downstairs","":"downstairs","session_id":"cc08105e-b5db-490d-a8d2-64e036755fce"} {"msg":"Max found gen is 42","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:55.494260665Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1301,"":"downstairs","":"downstairs","session_id":"cc08105e-b5db-490d-a8d2-64e036755fce"} {"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:55.494270443Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1301,"":"downstairs","":"downstairs","session_id":"cc08105e-b5db-490d-a8d2-64e036755fce"} {"msg":"Next flush: 77","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:55.494279642Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1301,"":"downstairs","":"downstairs","session_id":"cc08105e-b5db-490d-a8d2-64e036755fce"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:55.494289001Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1301,"":"downstairs","":"downstairs","session_id":"cc08105e-b5db-490d-a8d2-64e036755fce"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:55.494298989Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1301,"session_id":"cc08105e-b5db-490d-a8d2-64e036755fce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:55.494307678Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1301,"client":"0","":"downstairs","session_id":"cc08105e-b5db-490d-a8d2-64e036755fce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:55.494317596Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1301,"client":"1","":"downstairs","session_id":"cc08105e-b5db-490d-a8d2-64e036755fce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:55.494329045Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1301,"client":"2","":"downstairs","session_id":"cc08105e-b5db-490d-a8d2-64e036755fce"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:55.494338993Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1301,"session_id":"cc08105e-b5db-490d-a8d2-64e036755fce"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:55.494348212Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1301,"session_id":"cc08105e-b5db-490d-a8d2-64e036755fce"} {"msg":"0f4da21a-8eba-466b-8c42-984b9fa8f4d1 is now active with session: cc08105e-b5db-490d-a8d2-64e036755fce","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:55.4943581Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1301,"session_id":"cc08105e-b5db-490d-a8d2-64e036755fce"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:55.494367199Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1301,"session_id":"cc08105e-b5db-490d-a8d2-64e036755fce"} {"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:55.494376278Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1301} {"msg":"Activated sub_volume 0f4da21a-8eba-466b-8c42-984b9fa8f4d1","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:55.494384107Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1301} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:55.494392155Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1301,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:55.494691371Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1301,"task":"crutest"} 01/30 Write block 546 len 16384 data: 5 3 3 3 02/30 Write block 15 len 40960 data: 6 7 7 7 7 7 6 7 4 5 03/30 Write block 415 len 8192 data: 7 8 04/30 Read block 95 len 36864 05/30 Read block 513 len 20480 06/30 Write block 403 len 12288 data: 3 3 4 07/30 Write block 15 len 8192 data: 7 8 08/30 Read block 202 len 40960 09/30 Read block 460 len 8192 10/30 Read block 265 len 36864 11/30 Write block 405 len 36864 data: 5 5 5 4 4 7 9 8 7 12/30 Write block 178 len 8192 data: 5 3 13/30 Read block 142 len 32768 14/30 Read block 245 len 28672 15/30 Read block 42 len 4096 16/30 Flush 17/30 Write block 354 len 16384 data: 4 4 4 4 18/30 Read block 56 len 28672 19/30 Read block 574 len 40960 20/30 Write block 385 len 16384 data: 2 3 3 3 21/30 Write block 17 len 40960 data: 8 8 8 8 7 8 5 6 6 6 22/30 Write block 217 len 36864 data: 2 2 3 4 4 4 3 4 3 23/30 Read block 179 len 4096 24/30 Write block 354 len 32768 data: 5 5 5 5 4 5 5 4 25/30 Write block 372 len 16384 data: 4 3 3 3 26/30 Write block 111 len 36864 data: 3 3 3 3 4 3 3 3 3 27/30 Write block 301 len 24576 data: 2 2 2 2 4 4 28/30 Write block 300 len 8192 data: 2 3 29/30 Read block 257 len 20480 30/30 Write block 19 len 32768 data: 9 9 8 9 6 7 7 7 ---------------------------------------------------------------- Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Write 10 Done Done Done false 1002 Acked Write 2 Done Done Done false 1003 Acked Read 9 Done Done Done false 1004 Acked Read 5 Done Done Done false 1005 Acked Write 3 Done Done Done false 1006 Acked Write 2 Done Done Done false 1007 Acked Read 10 Done Done Done false 1008 Acked Read 2 Done Done Done false 1009 Acked Read 9 Done Done Done false 1010 Acked Write 9 Done Sent Done false 1011 Acked Write 2 Done Sent Done false 1012 Acked Read 8 Done Sent Done false 1013 Acked Read 7 Done Sent Done false 1014 Acked Read 1 Done Sent Done false 1015 Acked Flush 0 Done Sent Done false 1016 Acked Write 4 Done Sent Done false 1017 Acked Read 7 Done Sent Done false 1018 Acked Read 10 Done Sent Done false 1019 Acked Write 4 Done Sent Done false 1020 Acked Write 10 Done Sent Done false 1021 Acked Write 9 Done Sent Done false 1022 Acked Read 1 Done Sent Done false 1023 Acked Write 8 Done Sent Done false 1024 Acked Write 4 Done Sent Done false 1025 Acked Write 9 Done Sent Done false 1026 Acked Write 6 Done Sent Done false 1027 Acked Write 2 Done Sent Done false 1028 Acked Read 5 Done Sent Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 20 1 22 Done 29 10 29 68 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) None Some(JobId(1015)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:54:55.502714458Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1301} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:55.502913238Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1301,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 34 42 77 65 77 T T T <--- 1 020-039 42 41 42 77 76 77 T T T <--- 2 040-059 40 40 40 74 74 74 F F F 3 060-079 40 40 40 75 75 75 F F F 4 080-099 30 30 30 60 60 60 F F F 5 100-119 36 36 36 68 68 68 T F T <--- 6 120-139 32 32 32 63 63 63 F F F 7 140-159 34 34 34 65 65 65 F F F 8 160-179 42 34 42 77 65 77 F F F <--- 9 180-199 40 40 40 75 75 75 F F F 10 200-219 34 34 34 65 65 65 T F T <--- 11 220-239 41 41 41 76 76 76 T F T <--- 12 240-259 38 38 38 71 71 71 F F F 13 260-279 41 41 41 76 76 76 F F F 14 280-299 41 41 41 76 76 76 F F F 15 300-319 38 38 38 71 71 71 T F T <--- 16 320-339 36 36 36 67 67 67 F F F 17 340-359 40 40 40 73 73 73 T F T <--- 18 360-379 40 40 40 75 75 75 T F T <--- 19 380-399 38 38 38 70 70 70 T F T <--- 20 400-419 42 38 42 77 71 77 F T F <--- 21 420-439 39 39 39 72 72 72 F F F 22 440-459 39 39 39 72 72 72 F F F 23 460-479 40 40 40 74 74 74 F F F 24 480-499 41 41 41 76 76 76 F F F 25 500-519 41 41 41 76 76 76 F F F 26 520-539 41 41 41 76 76 76 F F F 27 540-559 42 41 42 77 76 77 F T F <--- 28 560-579 40 40 40 75 75 75 F F F 29 580-599 38 38 38 70 70 70 F F F Max gen: 42, Max flush: 77 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 21 Verifying data now No change after verify Loop: 21 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 43 43 43 78 78 78 F F F 1 020-039 43 43 43 78 78 78 F F F 2 040-059 40 40 40 74 74 74 F F F 3 060-079 40 40 40 75 75 75 F F F 4 080-099 30 30 30 60 60 60 F F F 5 100-119 43 43 43 78 78 78 F F F 6 120-139 32 32 32 63 63 63 F F F 7 140-159 34 34 34 65 65 65 F F F 8 160-179 42 42 42 77 77 77 F F F 9 180-199 40 40 40 75 75 75 F F F 10 200-219 43 43 43 78 78 78 F F F 11 220-239 43 43 43 78 78 78 F F F 12 240-259 38 38 38 71 71 71 F F F 13 260-279 41 41 41 76 76 76 F F F 14 280-299 41 41 41 76 76 76 F F F 15 300-319 43 43 43 78 78 78 F F F 16 320-339 36 36 36 67 67 67 F F F 17 340-359 43 43 43 78 78 78 F F F 18 360-379 43 43 43 78 78 78 F F F 19 380-399 43 43 43 78 78 78 F F F 20 400-419 42 42 42 77 77 77 F F F 21 420-439 39 39 39 72 72 72 F F F 22 440-459 39 39 39 72 72 72 F F F 23 460-479 40 40 40 74 74 74 F F F 24 480-499 41 41 41 76 76 76 F F F 25 500-519 41 41 41 76 76 76 F F F 26 520-539 41 41 41 76 76 76 F F F 27 540-559 42 42 42 77 77 77 F F F 28 560-579 40 40 40 75 75 75 F F F 29 580-599 38 38 38 70 70 70 F F F Max gen: 43, Max flush: 78 Begin loop 22 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:59.752058748Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1309,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:59.760146565Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1309,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:59.760427093Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1309} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:59.760453829Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1309} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:59.760462488Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1309} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:59.760474216Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1309} {"msg":"Crucible 2b2956d9-0c75-4c3b-86a4-b8a84c6eb8ce has session id: bddf1a0a-ec4d-4916-a708-e607b1221e1d","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:59.760775702Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1309,"session_id":"bddf1a0a-ec4d-4916-a708-e607b1221e1d"} {"msg":"Upstairs opts: Upstairs UUID: 2b2956d9-0c75-4c3b-86a4-b8a84c6eb8ce, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:59.760798419Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1309,"session_id":"bddf1a0a-ec4d-4916-a708-e607b1221e1d"} {"msg":"Crucible stats registered with UUID: 2b2956d9-0c75-4c3b-86a4-b8a84c6eb8ce","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:59.760808887Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1309,"session_id":"bddf1a0a-ec4d-4916-a708-e607b1221e1d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:54:59.760818116Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1309,"session_id":"bddf1a0a-ec4d-4916-a708-e607b1221e1d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:59.760960005Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1309,"":"io task","client":"1","":"downstairs","session_id":"bddf1a0a-ec4d-4916-a708-e607b1221e1d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:59.760976832Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1309,"":"io task","client":"0","":"downstairs","session_id":"bddf1a0a-ec4d-4916-a708-e607b1221e1d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:59.760986921Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1309,"":"io task","client":"2","":"downstairs","session_id":"bddf1a0a-ec4d-4916-a708-e607b1221e1d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:59.761151596Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1309,"task":"crutest"} {"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:59.761170144Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1309} {"msg":"2b2956d9-0c75-4c3b-86a4-b8a84c6eb8ce active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:59.76119359Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1309,"session_id":"bddf1a0a-ec4d-4916-a708-e607b1221e1d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:59.761203009Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1309,"client":"0","":"downstairs","session_id":"bddf1a0a-ec4d-4916-a708-e607b1221e1d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:59.761212947Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1309,"client":"1","":"downstairs","session_id":"bddf1a0a-ec4d-4916-a708-e607b1221e1d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:59.761222666Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1309,"client":"2","":"downstairs","session_id":"bddf1a0a-ec4d-4916-a708-e607b1221e1d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:59.761232305Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1309,"":"io task","client":"2","":"downstairs","session_id":"bddf1a0a-ec4d-4916-a708-e607b1221e1d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:59.761242333Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1309,"":"io task","client":"0","":"downstairs","session_id":"bddf1a0a-ec4d-4916-a708-e607b1221e1d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:59.761252592Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1309,"":"io task","client":"1","":"downstairs","session_id":"bddf1a0a-ec4d-4916-a708-e607b1221e1d"} {"msg":"ds_connection connected from Ok(127.0.0.1:48146)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:59.761304664Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1309,"":"io task","client":"2","":"downstairs","session_id":"bddf1a0a-ec4d-4916-a708-e607b1221e1d"} {"msg":"ds_connection connected from Ok(127.0.0.1:48954)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:59.761315522Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1309,"":"io task","client":"0","":"downstairs","session_id":"bddf1a0a-ec4d-4916-a708-e607b1221e1d"} {"msg":"ds_connection connected from Ok(127.0.0.1:46224)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:59.761325941Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1309,"":"io task","client":"1","":"downstairs","session_id":"bddf1a0a-ec4d-4916-a708-e607b1221e1d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:59.762067901Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1309,"client":"0","":"downstairs","session_id":"bddf1a0a-ec4d-4916-a708-e607b1221e1d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:59.762095587Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1309,"client":"1","":"downstairs","session_id":"bddf1a0a-ec4d-4916-a708-e607b1221e1d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:59.762168306Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1309,"client":"2","":"downstairs","session_id":"bddf1a0a-ec4d-4916-a708-e607b1221e1d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:59.76241244Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1309,"session_id":"bddf1a0a-ec4d-4916-a708-e607b1221e1d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:59.762435627Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1309,"session_id":"bddf1a0a-ec4d-4916-a708-e607b1221e1d"} {"msg":"[0]R flush_numbers[0..12]: [78, 78, 74, 75, 60, 78, 63, 65, 77, 75, 78, 78]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:59.762446455Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1309,"":"downstairs","":"downstairs","session_id":"bddf1a0a-ec4d-4916-a708-e607b1221e1d"} {"msg":"[0]R generation[0..12]: [43, 43, 40, 40, 30, 43, 32, 34, 42, 40, 43, 43]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:59.762457364Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1309,"":"downstairs","":"downstairs","session_id":"bddf1a0a-ec4d-4916-a708-e607b1221e1d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:59.762468722Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1309,"":"downstairs","":"downstairs","session_id":"bddf1a0a-ec4d-4916-a708-e607b1221e1d"} {"msg":"[1]R flush_numbers[0..12]: [78, 78, 74, 75, 60, 78, 63, 65, 77, 75, 78, 78]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:59.762479181Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1309,"":"downstairs","":"downstairs","session_id":"bddf1a0a-ec4d-4916-a708-e607b1221e1d"} {"msg":"[1]R generation[0..12]: [43, 43, 40, 40, 30, 43, 32, 34, 42, 40, 43, 43]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:59.762489609Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1309,"":"downstairs","":"downstairs","session_id":"bddf1a0a-ec4d-4916-a708-e607b1221e1d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:59.762500567Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1309,"":"downstairs","":"downstairs","session_id":"bddf1a0a-ec4d-4916-a708-e607b1221e1d"} {"msg":"[2]R flush_numbers[0..12]: [78, 78, 74, 75, 60, 78, 63, 65, 77, 75, 78, 78]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:59.762511126Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1309,"":"downstairs","":"downstairs","session_id":"bddf1a0a-ec4d-4916-a708-e607b1221e1d"} {"msg":"[2]R generation[0..12]: [43, 43, 40, 40, 30, 43, 32, 34, 42, 40, 43, 43]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:59.762524034Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1309,"":"downstairs","":"downstairs","session_id":"bddf1a0a-ec4d-4916-a708-e607b1221e1d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:59.762535012Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1309,"":"downstairs","":"downstairs","session_id":"bddf1a0a-ec4d-4916-a708-e607b1221e1d"} {"msg":"Max found gen is 44","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:59.762544721Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1309,"":"downstairs","":"downstairs","session_id":"bddf1a0a-ec4d-4916-a708-e607b1221e1d"} {"msg":"Generation requested: 44 >= found:44","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:59.762554489Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1309,"":"downstairs","":"downstairs","session_id":"bddf1a0a-ec4d-4916-a708-e607b1221e1d"} {"msg":"Next flush: 79","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:59.762563958Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1309,"":"downstairs","":"downstairs","session_id":"bddf1a0a-ec4d-4916-a708-e607b1221e1d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:59.762573447Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1309,"":"downstairs","":"downstairs","session_id":"bddf1a0a-ec4d-4916-a708-e607b1221e1d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:59.762583225Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1309,"session_id":"bddf1a0a-ec4d-4916-a708-e607b1221e1d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:59.762592054Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1309,"client":"0","":"downstairs","session_id":"bddf1a0a-ec4d-4916-a708-e607b1221e1d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:59.762601882Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1309,"client":"1","":"downstairs","session_id":"bddf1a0a-ec4d-4916-a708-e607b1221e1d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:59.762611651Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1309,"client":"2","":"downstairs","session_id":"bddf1a0a-ec4d-4916-a708-e607b1221e1d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:59.76262158Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1309,"session_id":"bddf1a0a-ec4d-4916-a708-e607b1221e1d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:59.762630838Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1309,"session_id":"bddf1a0a-ec4d-4916-a708-e607b1221e1d"} {"msg":"2b2956d9-0c75-4c3b-86a4-b8a84c6eb8ce is now active with session: bddf1a0a-ec4d-4916-a708-e607b1221e1d","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:59.762640957Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1309,"session_id":"bddf1a0a-ec4d-4916-a708-e607b1221e1d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:59.762649685Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1309,"session_id":"bddf1a0a-ec4d-4916-a708-e607b1221e1d"} {"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:59.762658734Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1309} {"msg":"Activated sub_volume 2b2956d9-0c75-4c3b-86a4-b8a84c6eb8ce","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:59.762666843Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1309} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:59.762676521Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1309,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:59.762971538Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1309,"task":"crutest"} 01/30 Write block 133 len 40960 data: 3 6 6 6 5 7 7 7 7 6 02/30 Write block 562 len 36864 data: 5 5 6 5 3 3 3 2 2 03/30 Write block 38 len 40960 data: 9 9 8 8 7 8 7 6 6 6 04/30 Write block 392 len 40960 data: 7 6 6 7 4 4 3 2 2 2 05/30 Flush 06/30 Write block 298 len 16384 data: 2 2 3 4 07/30 Write block 160 len 40960 data: 4 4 4 4 3 4 4 4 2 2 08/30 Read block 293 len 16384 09/30 Flush 10/30 Write block 267 len 20480 data: 8 5 5 5 6 11/30 Read block 315 len 28672 12/30 Write block 335 len 24576 data: 4 4 4 4 5 4 13/30 Write block 110 len 32768 data: 2 4 4 4 4 5 4 4 14/30 Write block 295 len 4096 data: 3 15/30 Read block 241 len 12288 16/30 Read block 310 len 24576 17/30 Write block 507 len 4096 data: 2 18/30 Read block 570 len 4096 19/30 Read block 309 len 28672 20/30 Write block 244 len 20480 data: 7 8 8 7 6 21/30 Read block 572 len 24576 22/30 Write block 409 len 12288 data: 5 8 10 23/30 Read block 150 len 4096 24/30 Read block 11 len 4096 25/30 Write block 131 len 20480 data: 3 3 4 7 7 26/30 Read block 80 len 4096 27/30 Write block 101 len 28672 data: 5 3 3 3 4 3 4 28/30 Read block 514 len 8192 29/30 Write block 534 len 36864 data: 4 4 3 3 4 4 4 4 4 30/30 Read block 52 len 20480 ---------------------------------------------------------------- Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Write 9 Done Sent Done false 1002 Acked Write 10 Done Sent Done false 1003 Acked Write 10 Done Sent Done false 1004 Acked Flush 0 Done Sent Done false 1005 Acked Write 4 Done Sent Done false 1006 Acked Write 10 Done Sent Done false 1007 Acked Read 4 Done Sent Done false 1008 Acked Flush 0 Done Sent Done false 1009 Acked Write 5 Done Sent Done false 1010 Acked Read 7 Done Sent Done false 1011 Acked Write 6 Done Sent Done false 1012 Acked Write 8 Done Sent Done false 1013 Acked Write 1 Done Sent Done false 1014 Acked Read 3 Done Sent Done false 1015 Acked Read 6 Done Sent Done false 1016 Acked Write 1 Done Sent Done false 1017 Acked Read 1 Done Sent Done false 1018 Acked Read 7 Done Sent Done false 1019 Acked Write 5 Done Sent Done false 1020 Acked Read 6 Done Sent Done false 1021 Acked Write 3 Done Sent Done false 1022 Acked Read 1 Done Sent Done false 1023 Acked Read 1 Done Sent Done false 1024 Acked Write 5 Done Sent Done false 1025 Acked Read 1 Done Sent Done false 1026 Acked Write 7 Done Sent Done false 1027 Acked Read 2 Done Sent Done false 1028 Acked Write 9 Done Sent Done false 1029 Acked Read 5 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 29 0 29 Done 30 1 30 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1008)) None Some(JobId(1008)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:54:59.771457996Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1309} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:54:59.771642589Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1309,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 43 43 43 78 78 78 F F F 1 020-039 44 43 44 79 78 79 F F F <--- 2 040-059 44 40 44 79 74 79 F F F <--- 3 060-079 40 40 40 75 75 75 F F F 4 080-099 30 30 30 60 60 60 F F F 5 100-119 43 43 43 78 78 78 T F T <--- 6 120-139 44 32 44 79 63 79 T T T <--- 7 140-159 44 34 44 79 65 79 F T F <--- 8 160-179 44 42 44 80 77 80 F F F <--- 9 180-199 40 40 40 75 75 75 F F F 10 200-219 43 43 43 78 78 78 F F F 11 220-239 43 43 43 78 78 78 F F F 12 240-259 38 38 38 71 71 71 T F T <--- 13 260-279 41 41 41 76 76 76 T F T <--- 14 280-299 44 41 44 80 76 80 T F T <--- 15 300-319 44 43 44 80 78 80 F F F <--- 16 320-339 36 36 36 67 67 67 T F T <--- 17 340-359 43 43 43 78 78 78 T F T <--- 18 360-379 43 43 43 78 78 78 F F F 19 380-399 44 43 44 79 78 79 F F F <--- 20 400-419 44 42 44 79 77 79 T F T <--- 21 420-439 39 39 39 72 72 72 F F F 22 440-459 39 39 39 72 72 72 F F F 23 460-479 40 40 40 74 74 74 F F F 24 480-499 41 41 41 76 76 76 F F F 25 500-519 41 41 41 76 76 76 T F T <--- 26 520-539 41 41 41 76 76 76 T F T <--- 27 540-559 42 42 42 77 77 77 T F T <--- 28 560-579 44 40 44 79 75 79 F F F <--- 29 580-599 38 38 38 70 70 70 F F F Max gen: 44, Max flush: 80 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 22 Verifying data now No change after verify Loop: 22 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 43 43 43 78 78 78 F F F 1 020-039 44 44 44 79 79 79 F F F 2 040-059 44 44 44 79 79 79 F F F 3 060-079 40 40 40 75 75 75 F F F 4 080-099 30 30 30 60 60 60 F F F 5 100-119 45 45 45 81 81 81 F F F 6 120-139 45 45 45 81 81 81 F F F 7 140-159 44 44 44 79 79 79 F F F 8 160-179 44 44 44 80 80 80 F F F 9 180-199 40 40 40 75 75 75 F F F 10 200-219 43 43 43 78 78 78 F F F 11 220-239 43 43 43 78 78 78 F F F 12 240-259 45 45 45 81 81 81 F F F 13 260-279 45 45 45 81 81 81 F F F 14 280-299 45 45 45 81 81 81 F F F 15 300-319 44 44 44 80 80 80 F F F 16 320-339 45 45 45 81 81 81 F F F 17 340-359 45 45 45 81 81 81 F F F 18 360-379 43 43 43 78 78 78 F F F 19 380-399 44 44 44 79 79 79 F F F 20 400-419 45 45 45 81 81 81 F F F 21 420-439 39 39 39 72 72 72 F F F 22 440-459 39 39 39 72 72 72 F F F 23 460-479 40 40 40 74 74 74 F F F 24 480-499 41 41 41 76 76 76 F F F 25 500-519 45 45 45 81 81 81 F F F 26 520-539 45 45 45 81 81 81 F F F 27 540-559 45 45 45 81 81 81 F F F 28 560-579 44 44 44 79 79 79 F F F 29 580-599 38 38 38 70 70 70 F F F Max gen: 45, Max flush: 81 Begin loop 23 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:04.034528541Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1317,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:04.042573595Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1317,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:04.042866501Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1317} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:04.042890548Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1317} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:04.042899496Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1317} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:04.042911075Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1317} {"msg":"Crucible 17c31a8f-5e7b-4870-98ed-df83beae77d8 has session id: a1fe5ebc-3431-43cd-acae-e1ad6bbfb88c","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:04.043272491Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1317,"session_id":"a1fe5ebc-3431-43cd-acae-e1ad6bbfb88c"} {"msg":"Upstairs opts: Upstairs UUID: 17c31a8f-5e7b-4870-98ed-df83beae77d8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:04.043294268Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1317,"session_id":"a1fe5ebc-3431-43cd-acae-e1ad6bbfb88c"} {"msg":"Crucible stats registered with UUID: 17c31a8f-5e7b-4870-98ed-df83beae77d8","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:04.043304517Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1317,"session_id":"a1fe5ebc-3431-43cd-acae-e1ad6bbfb88c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:55:04.043313945Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1317,"session_id":"a1fe5ebc-3431-43cd-acae-e1ad6bbfb88c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:04.043464523Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1317,"":"io task","client":"1","":"downstairs","session_id":"a1fe5ebc-3431-43cd-acae-e1ad6bbfb88c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:04.043495109Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1317,"":"io task","client":"0","":"downstairs","session_id":"a1fe5ebc-3431-43cd-acae-e1ad6bbfb88c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:04.043508787Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1317,"":"io task","client":"2","":"downstairs","session_id":"a1fe5ebc-3431-43cd-acae-e1ad6bbfb88c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:04.04362332Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1317,"task":"crutest"} {"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:04.043637408Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1317} {"msg":"17c31a8f-5e7b-4870-98ed-df83beae77d8 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:04.043674512Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1317,"session_id":"a1fe5ebc-3431-43cd-acae-e1ad6bbfb88c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:04.043683971Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1317,"client":"0","":"downstairs","session_id":"a1fe5ebc-3431-43cd-acae-e1ad6bbfb88c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:04.043693849Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1317,"client":"1","":"downstairs","session_id":"a1fe5ebc-3431-43cd-acae-e1ad6bbfb88c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:04.043703818Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1317,"client":"2","":"downstairs","session_id":"a1fe5ebc-3431-43cd-acae-e1ad6bbfb88c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:04.043713486Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1317,"":"io task","client":"2","":"downstairs","session_id":"a1fe5ebc-3431-43cd-acae-e1ad6bbfb88c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:04.043723375Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1317,"":"io task","client":"0","":"downstairs","session_id":"a1fe5ebc-3431-43cd-acae-e1ad6bbfb88c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:04.043733493Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1317,"":"io task","client":"1","":"downstairs","session_id":"a1fe5ebc-3431-43cd-acae-e1ad6bbfb88c"} {"msg":"ds_connection connected from Ok(127.0.0.1:65056)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:04.043770018Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1317,"":"io task","client":"0","":"downstairs","session_id":"a1fe5ebc-3431-43cd-acae-e1ad6bbfb88c"} {"msg":"ds_connection connected from Ok(127.0.0.1:47776)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:04.043789175Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1317,"":"io task","client":"1","":"downstairs","session_id":"a1fe5ebc-3431-43cd-acae-e1ad6bbfb88c"} {"msg":"ds_connection connected from Ok(127.0.0.1:59283)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:04.043799834Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1317,"":"io task","client":"2","":"downstairs","session_id":"a1fe5ebc-3431-43cd-acae-e1ad6bbfb88c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:04.044480503Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1317,"client":"2","":"downstairs","session_id":"a1fe5ebc-3431-43cd-acae-e1ad6bbfb88c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:04.044508589Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1317,"client":"1","":"downstairs","session_id":"a1fe5ebc-3431-43cd-acae-e1ad6bbfb88c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:04.044527146Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1317,"client":"0","":"downstairs","session_id":"a1fe5ebc-3431-43cd-acae-e1ad6bbfb88c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:04.044776899Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1317,"session_id":"a1fe5ebc-3431-43cd-acae-e1ad6bbfb88c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:04.044812064Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1317,"session_id":"a1fe5ebc-3431-43cd-acae-e1ad6bbfb88c"} {"msg":"[0]R flush_numbers[0..12]: [78, 79, 79, 75, 60, 81, 81, 79, 80, 75, 78, 78]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:04.044847909Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1317,"":"downstairs","":"downstairs","session_id":"a1fe5ebc-3431-43cd-acae-e1ad6bbfb88c"} {"msg":"[0]R generation[0..12]: [43, 44, 44, 40, 30, 45, 45, 44, 44, 40, 43, 43]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:04.044858907Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1317,"":"downstairs","":"downstairs","session_id":"a1fe5ebc-3431-43cd-acae-e1ad6bbfb88c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:04.044870056Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1317,"":"downstairs","":"downstairs","session_id":"a1fe5ebc-3431-43cd-acae-e1ad6bbfb88c"} {"msg":"[1]R flush_numbers[0..12]: [78, 79, 79, 75, 60, 81, 81, 79, 80, 75, 78, 78]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:04.044880604Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1317,"":"downstairs","":"downstairs","session_id":"a1fe5ebc-3431-43cd-acae-e1ad6bbfb88c"} {"msg":"[1]R generation[0..12]: [43, 44, 44, 40, 30, 45, 45, 44, 44, 40, 43, 43]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:04.044890903Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1317,"":"downstairs","":"downstairs","session_id":"a1fe5ebc-3431-43cd-acae-e1ad6bbfb88c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:04.044901721Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1317,"":"downstairs","":"downstairs","session_id":"a1fe5ebc-3431-43cd-acae-e1ad6bbfb88c"} {"msg":"[2]R flush_numbers[0..12]: [78, 79, 79, 75, 60, 81, 81, 79, 80, 75, 78, 78]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:04.04491202Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1317,"":"downstairs","":"downstairs","session_id":"a1fe5ebc-3431-43cd-acae-e1ad6bbfb88c"} {"msg":"[2]R generation[0..12]: [43, 44, 44, 40, 30, 45, 45, 44, 44, 40, 43, 43]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:04.044922728Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1317,"":"downstairs","":"downstairs","session_id":"a1fe5ebc-3431-43cd-acae-e1ad6bbfb88c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:04.044933686Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1317,"":"downstairs","":"downstairs","session_id":"a1fe5ebc-3431-43cd-acae-e1ad6bbfb88c"} {"msg":"Max found gen is 46","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:04.044943105Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1317,"":"downstairs","":"downstairs","session_id":"a1fe5ebc-3431-43cd-acae-e1ad6bbfb88c"} {"msg":"Generation requested: 46 >= found:46","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:04.044952724Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1317,"":"downstairs","":"downstairs","session_id":"a1fe5ebc-3431-43cd-acae-e1ad6bbfb88c"} {"msg":"Next flush: 82","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:04.044962042Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1317,"":"downstairs","":"downstairs","session_id":"a1fe5ebc-3431-43cd-acae-e1ad6bbfb88c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:04.044972871Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1317,"":"downstairs","":"downstairs","session_id":"a1fe5ebc-3431-43cd-acae-e1ad6bbfb88c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:04.044982439Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1317,"session_id":"a1fe5ebc-3431-43cd-acae-e1ad6bbfb88c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:04.044991268Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1317,"client":"0","":"downstairs","session_id":"a1fe5ebc-3431-43cd-acae-e1ad6bbfb88c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:04.045000766Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1317,"client":"1","":"downstairs","session_id":"a1fe5ebc-3431-43cd-acae-e1ad6bbfb88c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:04.045010675Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1317,"client":"2","":"downstairs","session_id":"a1fe5ebc-3431-43cd-acae-e1ad6bbfb88c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:04.045020224Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1317,"session_id":"a1fe5ebc-3431-43cd-acae-e1ad6bbfb88c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:04.045029212Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1317,"session_id":"a1fe5ebc-3431-43cd-acae-e1ad6bbfb88c"} {"msg":"17c31a8f-5e7b-4870-98ed-df83beae77d8 is now active with session: a1fe5ebc-3431-43cd-acae-e1ad6bbfb88c","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:04.045039121Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1317,"session_id":"a1fe5ebc-3431-43cd-acae-e1ad6bbfb88c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:04.045047759Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1317,"session_id":"a1fe5ebc-3431-43cd-acae-e1ad6bbfb88c"} {"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:04.045057048Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1317} {"msg":"Activated sub_volume 17c31a8f-5e7b-4870-98ed-df83beae77d8","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:04.045065297Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1317} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:04.045073416Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1317,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:04.045344316Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1317,"task":"crutest"} 01/30 Flush 02/30 Write block 20 len 24576 data: 10 9 10 7 8 8 03/30 Read block 293 len 16384 04/30 Read block 520 len 40960 05/30 Write block 356 len 12288 data: 6 6 5 06/30 Read block 418 len 20480 07/30 Read block 429 len 32768 08/30 Write block 106 len 36864 data: 4 5 4 2 3 5 5 5 5 09/30 Write block 207 len 36864 data: 8 6 4 3 3 4 4 4 3 10/30 Read block 398 len 4096 11/30 Read block 196 len 8192 12/30 Write block 413 len 16384 data: 8 6 8 9 13/30 Flush 14/30 Flush 15/30 Flush 16/30 Write block 462 len 28672 data: 4 4 4 4 5 5 6 17/30 Write block 274 len 8192 data: 5 7 18/30 Write block 471 len 20480 data: 7 6 4 6 6 19/30 Read block 496 len 28672 20/30 Flush 21/30 Write block 92 len 20480 data: 4 4 3 4 4 22/30 Flush 23/30 Write block 278 len 32768 data: 6 4 4 3 3 4 5 5 24/30 Read block 567 len 20480 25/30 Write block 69 len 36864 data: 4 5 5 5 5 5 4 4 4 26/30 Read block 259 len 36864 27/30 Write block 79 len 8192 data: 4 3 28/30 Read block 276 len 16384 29/30 Read block 5 len 12288 30/30 Write block 309 len 12288 data: 4 6 8 ---------------------------------------------------------------- Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 6 Done Done Done false 1002 Acked Read 4 Done Sent Done false 1003 Acked Read 10 Done Sent Done false 1004 Acked Write 3 Done Sent Done false 1005 Acked Read 5 Done Sent Done false 1006 Acked Read 8 Done Sent Done false 1007 Acked Write 9 Done Sent Done false 1008 Acked Write 9 Done Sent Done false 1009 Acked Read 1 Done Sent Done false 1010 Acked Read 2 Done Sent Done false 1011 Acked Write 4 Done Sent Done false 1012 Acked Flush 0 Done Sent Done false 1013 Acked Flush 0 Done Sent Done false 1014 Acked Flush 0 Done Sent Done false 1015 Acked Write 7 Done Sent Done false 1016 Acked Write 2 Done Sent Done false 1017 Acked Write 5 Done Sent Done false 1018 Acked Read 7 Done Sent Done false 1019 Acked Flush 0 Done Sent Done false 1020 Acked Write 5 Done Sent Done false 1021 Acked Flush 0 Done Sent Done false 1022 Acked Write 8 Done Sent Done false 1023 Acked Read 5 Done Sent Done false 1024 Acked Write 9 Done Sent Done false 1025 Acked Read 9 Done Sent Done false 1026 Acked Write 2 Done Sent Done false 1027 Acked Read 4 Done Sent Done false 1028 Acked Read 3 Done Sent Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 28 1 30 Done 28 1 28 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1000)) Some(JobId(1021)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:55:04.054112812Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1317} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:04.054309723Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1317,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 43 43 43 78 78 78 F F F 1 020-039 46 44 46 83 79 83 F T F <--- 2 040-059 44 44 44 79 79 79 F F F 3 060-079 40 40 40 75 75 75 T F T <--- 4 080-099 46 30 46 87 60 87 T F T <--- 5 100-119 46 45 46 83 81 83 F F F <--- 6 120-139 45 45 45 81 81 81 F F F 7 140-159 44 44 44 79 79 79 F F F 8 160-179 44 44 44 80 80 80 F F F 9 180-199 40 40 40 75 75 75 F F F 10 200-219 46 43 46 83 78 83 F F F <--- 11 220-239 43 43 43 78 78 78 F F F 12 240-259 45 45 45 81 81 81 F F F 13 260-279 46 45 46 86 81 86 T F T <--- 14 280-299 45 45 45 81 81 81 T F T <--- 15 300-319 44 44 44 80 80 80 T F T <--- 16 320-339 45 45 45 81 81 81 F F F 17 340-359 46 45 46 83 81 83 F F F <--- 18 360-379 43 43 43 78 78 78 F F F 19 380-399 44 44 44 79 79 79 F F F 20 400-419 46 45 46 83 81 83 F F F <--- 21 420-439 39 39 39 72 72 72 F F F 22 440-459 39 39 39 72 72 72 F F F 23 460-479 46 40 46 86 74 86 F F F <--- 24 480-499 41 41 41 76 76 76 F F F 25 500-519 45 45 45 81 81 81 F F F 26 520-539 45 45 45 81 81 81 F F F 27 540-559 45 45 45 81 81 81 F F F 28 560-579 44 44 44 79 79 79 F F F 29 580-599 38 38 38 70 70 70 F F F Max gen: 46, Max flush: 87 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 23 Verifying data now No change after verify Loop: 23 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 43 43 43 78 78 78 F F F 1 020-039 46 46 46 83 83 83 F F F 2 040-059 44 44 44 79 79 79 F F F 3 060-079 47 47 47 88 88 88 F F F 4 080-099 47 47 47 88 88 88 F F F 5 100-119 46 46 46 83 83 83 F F F 6 120-139 45 45 45 81 81 81 F F F 7 140-159 44 44 44 79 79 79 F F F 8 160-179 44 44 44 80 80 80 F F F 9 180-199 40 40 40 75 75 75 F F F 10 200-219 46 46 46 83 83 83 F F F 11 220-239 43 43 43 78 78 78 F F F 12 240-259 45 45 45 81 81 81 F F F 13 260-279 47 47 47 88 88 88 F F F 14 280-299 47 47 47 88 88 88 F F F 15 300-319 47 47 47 88 88 88 F F F 16 320-339 45 45 45 81 81 81 F F F 17 340-359 46 46 46 83 83 83 F F F 18 360-379 43 43 43 78 78 78 F F F 19 380-399 44 44 44 79 79 79 F F F 20 400-419 46 46 46 83 83 83 F F F 21 420-439 39 39 39 72 72 72 F F F 22 440-459 39 39 39 72 72 72 F F F 23 460-479 46 46 46 86 86 86 F F F 24 480-499 41 41 41 76 76 76 F F F 25 500-519 45 45 45 81 81 81 F F F 26 520-539 45 45 45 81 81 81 F F F 27 540-559 45 45 45 81 81 81 F F F 28 560-579 44 44 44 79 79 79 F F F 29 580-599 38 38 38 70 70 70 F F F Max gen: 47, Max flush: 88 Begin loop 24 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:08.299394262Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1325,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:08.307428597Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1325,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:08.307721383Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1325} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:08.3077459Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1325} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:08.307757908Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1325} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:08.307769506Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1325} {"msg":"Crucible c1eed33e-61e6-421f-aef1-75aca67c1e9f has session id: 920a5a90-ae91-4615-aa46-6f5d54080cec","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:08.308121834Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1325,"session_id":"920a5a90-ae91-4615-aa46-6f5d54080cec"} {"msg":"Upstairs opts: Upstairs UUID: c1eed33e-61e6-421f-aef1-75aca67c1e9f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:08.308169207Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1325,"session_id":"920a5a90-ae91-4615-aa46-6f5d54080cec"} {"msg":"Crucible stats registered with UUID: c1eed33e-61e6-421f-aef1-75aca67c1e9f","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:08.308180956Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1325,"session_id":"920a5a90-ae91-4615-aa46-6f5d54080cec"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:55:08.308190224Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1325,"session_id":"920a5a90-ae91-4615-aa46-6f5d54080cec"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:08.308337433Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1325,"":"io task","client":"0","":"downstairs","session_id":"920a5a90-ae91-4615-aa46-6f5d54080cec"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:08.30835398Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1325,"":"io task","client":"1","":"downstairs","session_id":"920a5a90-ae91-4615-aa46-6f5d54080cec"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:08.308364179Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1325,"":"io task","client":"2","":"downstairs","session_id":"920a5a90-ae91-4615-aa46-6f5d54080cec"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:08.308533944Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1325,"task":"crutest"} {"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:08.308549571Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1325} {"msg":"c1eed33e-61e6-421f-aef1-75aca67c1e9f active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:08.308581207Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1325,"session_id":"920a5a90-ae91-4615-aa46-6f5d54080cec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:08.308590625Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1325,"client":"0","":"downstairs","session_id":"920a5a90-ae91-4615-aa46-6f5d54080cec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:08.308600214Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1325,"client":"1","":"downstairs","session_id":"920a5a90-ae91-4615-aa46-6f5d54080cec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:08.308609962Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1325,"client":"2","":"downstairs","session_id":"920a5a90-ae91-4615-aa46-6f5d54080cec"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:08.308621641Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1325,"":"io task","client":"2","":"downstairs","session_id":"920a5a90-ae91-4615-aa46-6f5d54080cec"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:08.308631739Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1325,"":"io task","client":"0","":"downstairs","session_id":"920a5a90-ae91-4615-aa46-6f5d54080cec"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:08.308641818Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1325,"":"io task","client":"1","":"downstairs","session_id":"920a5a90-ae91-4615-aa46-6f5d54080cec"} {"msg":"ds_connection connected from Ok(127.0.0.1:63816)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:08.308687181Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1325,"":"io task","client":"2","":"downstairs","session_id":"920a5a90-ae91-4615-aa46-6f5d54080cec"} {"msg":"ds_connection connected from Ok(127.0.0.1:43252)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:08.308724155Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1325,"":"io task","client":"1","":"downstairs","session_id":"920a5a90-ae91-4615-aa46-6f5d54080cec"} {"msg":"ds_connection connected from Ok(127.0.0.1:40037)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:08.308735224Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1325,"":"io task","client":"0","":"downstairs","session_id":"920a5a90-ae91-4615-aa46-6f5d54080cec"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:08.309384368Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1325,"client":"2","":"downstairs","session_id":"920a5a90-ae91-4615-aa46-6f5d54080cec"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:08.309409684Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1325,"client":"1","":"downstairs","session_id":"920a5a90-ae91-4615-aa46-6f5d54080cec"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:08.309455588Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1325,"client":"0","":"downstairs","session_id":"920a5a90-ae91-4615-aa46-6f5d54080cec"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:08.309686354Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1325,"session_id":"920a5a90-ae91-4615-aa46-6f5d54080cec"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:08.309696222Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1325,"session_id":"920a5a90-ae91-4615-aa46-6f5d54080cec"} {"msg":"[0]R flush_numbers[0..12]: [78, 83, 79, 88, 88, 83, 81, 79, 80, 75, 83, 78]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:08.309809375Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1325,"":"downstairs","":"downstairs","session_id":"920a5a90-ae91-4615-aa46-6f5d54080cec"} {"msg":"[0]R generation[0..12]: [43, 46, 44, 47, 47, 46, 45, 44, 44, 40, 46, 43]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:08.309836621Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1325,"":"downstairs","":"downstairs","session_id":"920a5a90-ae91-4615-aa46-6f5d54080cec"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:08.30984875Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1325,"":"downstairs","":"downstairs","session_id":"920a5a90-ae91-4615-aa46-6f5d54080cec"} {"msg":"[1]R flush_numbers[0..12]: [78, 83, 79, 88, 88, 83, 81, 79, 80, 75, 83, 78]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:08.309859758Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1325,"":"downstairs","":"downstairs","session_id":"920a5a90-ae91-4615-aa46-6f5d54080cec"} {"msg":"[1]R generation[0..12]: [43, 46, 44, 47, 47, 46, 45, 44, 44, 40, 46, 43]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:08.309872846Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1325,"":"downstairs","":"downstairs","session_id":"920a5a90-ae91-4615-aa46-6f5d54080cec"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:08.309884024Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1325,"":"downstairs","":"downstairs","session_id":"920a5a90-ae91-4615-aa46-6f5d54080cec"} {"msg":"[2]R flush_numbers[0..12]: [78, 83, 79, 88, 88, 83, 81, 79, 80, 75, 83, 78]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:08.309894483Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1325,"":"downstairs","":"downstairs","session_id":"920a5a90-ae91-4615-aa46-6f5d54080cec"} {"msg":"[2]R generation[0..12]: [43, 46, 44, 47, 47, 46, 45, 44, 44, 40, 46, 43]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:08.309904961Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1325,"":"downstairs","":"downstairs","session_id":"920a5a90-ae91-4615-aa46-6f5d54080cec"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:08.30991603Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1325,"":"downstairs","":"downstairs","session_id":"920a5a90-ae91-4615-aa46-6f5d54080cec"} {"msg":"Max found gen is 48","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:08.309925708Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1325,"":"downstairs","":"downstairs","session_id":"920a5a90-ae91-4615-aa46-6f5d54080cec"} {"msg":"Generation requested: 48 >= found:48","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:08.309935487Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1325,"":"downstairs","":"downstairs","session_id":"920a5a90-ae91-4615-aa46-6f5d54080cec"} {"msg":"Next flush: 89","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:08.309944905Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1325,"":"downstairs","":"downstairs","session_id":"920a5a90-ae91-4615-aa46-6f5d54080cec"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:08.309954414Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1325,"":"downstairs","":"downstairs","session_id":"920a5a90-ae91-4615-aa46-6f5d54080cec"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:08.309964322Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1325,"session_id":"920a5a90-ae91-4615-aa46-6f5d54080cec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:08.309973101Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1325,"client":"0","":"downstairs","session_id":"920a5a90-ae91-4615-aa46-6f5d54080cec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:08.30998299Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1325,"client":"1","":"downstairs","session_id":"920a5a90-ae91-4615-aa46-6f5d54080cec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:08.309992668Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1325,"client":"2","":"downstairs","session_id":"920a5a90-ae91-4615-aa46-6f5d54080cec"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:08.310002877Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1325,"session_id":"920a5a90-ae91-4615-aa46-6f5d54080cec"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:08.310012295Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1325,"session_id":"920a5a90-ae91-4615-aa46-6f5d54080cec"} {"msg":"c1eed33e-61e6-421f-aef1-75aca67c1e9f is now active with session: 920a5a90-ae91-4615-aa46-6f5d54080cec","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:08.310023694Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1325,"session_id":"920a5a90-ae91-4615-aa46-6f5d54080cec"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:08.310032342Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1325,"session_id":"920a5a90-ae91-4615-aa46-6f5d54080cec"} {"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:08.310041461Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1325} {"msg":"Activated sub_volume c1eed33e-61e6-421f-aef1-75aca67c1e9f","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:08.31004957Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1325} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:08.310057779Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1325,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:08.310342237Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1325,"task":"crutest"} 01/30 Flush 02/30 Write block 533 len 4096 data: 5 03/30 Read block 255 len 32768 04/30 Flush 05/30 Flush 06/30 Write block 409 len 20480 data: 6 9 11 9 9 07/30 Read block 364 len 12288 08/30 Read block 550 len 16384 09/30 Write block 278 len 12288 data: 7 5 5 10/30 Write block 435 len 8192 data: 5 4 11/30 Write block 404 len 8192 data: 4 6 12/30 Write block 472 len 16384 data: 7 5 7 7 13/30 Read block 314 len 40960 14/30 Read block 366 len 40960 15/30 Write block 536 len 4096 data: 4 16/30 Flush 17/30 Write block 191 len 8192 data: 9 9 18/30 Write block 411 len 24576 data: 12 10 10 7 9 10 19/30 Write block 322 len 40960 data: 5 6 6 7 7 6 5 5 5 5 20/30 Write block 514 len 24576 data: 6 7 5 6 8 8 21/30 Flush 22/30 Write block 567 len 36864 data: 4 4 3 3 2 2 3 4 4 23/30 Write block 422 len 36864 data: 4 4 4 5 4 5 5 5 4 24/30 Flush 25/30 Write block 192 len 36864 data: 10 9 8 6 5 6 6 5 4 26/30 Write block 316 len 40960 data: 7 5 4 5 6 5 6 7 7 8 27/30 Write block 149 len 20480 data: 5 5 5 5 6 28/30 Read block 223 len 36864 29/30 Read block 423 len 8192 30/30 Read block 560 len 16384 ---------------------------------------------------------------- Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Flush 0 Done Sent Done false 1001 Acked Write 1 Done Sent Done false 1002 Acked Read 8 Done Sent Done false 1003 Acked Flush 0 Done Sent Done false 1004 Acked Flush 0 Done Sent Done false 1005 Acked Write 5 Done Sent Done false 1006 Acked Read 3 Done Sent Done false 1007 Acked Read 4 Done Sent Done false 1008 Acked Write 3 Done Sent Done false 1009 Acked Write 2 Done Sent Done false 1010 Acked Write 2 Done Sent Done false 1011 Acked Write 4 Done Sent Done false 1012 Acked Read 10 Done Sent Done false 1013 Acked Read 10 Done Sent Done false 1014 Acked Write 1 Done Sent Done false 1015 Acked Flush 0 Done Sent Done false 1016 Acked Write 2 Done Sent Done false 1017 Acked Write 6 Done Sent Done false 1018 Acked Write 10 Done Sent Done false 1019 Acked Write 6 Done Sent Done false 1020 Acked Flush 0 Done Sent Done false 1021 Acked Write 9 Done Sent Done false 1022 Acked Write 9 Done Sent Done false 1023 Acked Flush 0 Done Sent Done false 1024 Acked Write 9 Done Sent Done false 1025 Acked Write 10 Done Sent Done false 1026 Acked Write 5 Done Sent Done false 1027 Acked Read 9 Done Sent Done false 1028 Acked Read 2 Done Sent Done false 1029 Acked Read 4 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 30 0 30 Done 30 0 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) None Some(JobId(1023)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:55:08.31817975Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1325} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:08.318347576Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1325,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 43 43 43 78 78 78 F F F 1 020-039 46 46 46 83 83 83 F F F 2 040-059 44 44 44 79 79 79 F F F 3 060-079 47 47 47 88 88 88 F F F 4 080-099 47 47 47 88 88 88 F F F 5 100-119 46 46 46 83 83 83 F F F 6 120-139 45 45 45 81 81 81 F F F 7 140-159 44 44 44 79 79 79 T F T <--- 8 160-179 44 44 44 80 80 80 F F F 9 180-199 48 40 48 93 75 93 T F T <--- 10 200-219 46 46 46 83 83 83 T F T <--- 11 220-239 43 43 43 78 78 78 F F F 12 240-259 45 45 45 81 81 81 F F F 13 260-279 48 47 48 92 88 92 F F F <--- 14 280-299 48 47 48 92 88 92 F F F <--- 15 300-319 47 47 47 88 88 88 T F T <--- 16 320-339 48 45 48 93 81 93 T F T <--- 17 340-359 46 46 46 83 83 83 F F F 18 360-379 43 43 43 78 78 78 F F F 19 380-399 44 44 44 79 79 79 F F F 20 400-419 48 46 48 93 83 93 F F F <--- 21 420-439 48 39 48 94 72 94 F F F <--- 22 440-459 39 39 39 72 72 72 F F F 23 460-479 48 46 48 92 86 92 F F F <--- 24 480-499 41 41 41 76 76 76 F F F 25 500-519 48 45 48 93 81 93 F F F <--- 26 520-539 48 45 48 92 81 92 F F F <--- 27 540-559 45 45 45 81 81 81 F F F 28 560-579 48 44 48 94 79 94 F F F <--- 29 580-599 38 38 38 70 70 70 F F F Max gen: 48, Max flush: 94 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 24 Verifying data now No change after verify Loop: 24 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 43 43 43 78 78 78 F F F 1 020-039 46 46 46 83 83 83 F F F 2 040-059 44 44 44 79 79 79 F F F 3 060-079 47 47 47 88 88 88 F F F 4 080-099 47 47 47 88 88 88 F F F 5 100-119 46 46 46 83 83 83 F F F 6 120-139 45 45 45 81 81 81 F F F 7 140-159 49 49 49 95 95 95 F F F 8 160-179 44 44 44 80 80 80 F F F 9 180-199 49 49 49 95 95 95 F F F 10 200-219 49 49 49 95 95 95 F F F 11 220-239 43 43 43 78 78 78 F F F 12 240-259 45 45 45 81 81 81 F F F 13 260-279 48 48 48 92 92 92 F F F 14 280-299 48 48 48 92 92 92 F F F 15 300-319 49 49 49 95 95 95 F F F 16 320-339 49 49 49 95 95 95 F F F 17 340-359 46 46 46 83 83 83 F F F 18 360-379 43 43 43 78 78 78 F F F 19 380-399 44 44 44 79 79 79 F F F 20 400-419 48 48 48 93 93 93 F F F 21 420-439 48 48 48 94 94 94 F F F 22 440-459 39 39 39 72 72 72 F F F 23 460-479 48 48 48 92 92 92 F F F 24 480-499 41 41 41 76 76 76 F F F 25 500-519 48 48 48 93 93 93 F F F 26 520-539 48 48 48 92 92 92 F F F 27 540-559 45 45 45 81 81 81 F F F 28 560-579 48 48 48 94 94 94 F F F 29 580-599 38 38 38 70 70 70 F F F Max gen: 49, Max flush: 95 Begin loop 25 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:12.573326765Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1333,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:55:12.580431076Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1333,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:12.580483679Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1333,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:12.587501553Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1333,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:12.587767114Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1333} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:12.58779221Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1333} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:12.587800749Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1333} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:12.587812257Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1333} {"msg":"Crucible 6a3ca455-08b9-4dbe-8019-0366253bea7a has session id: b30686e4-2885-4734-8397-449fe1ab9918","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:12.588176534Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1333,"session_id":"b30686e4-2885-4734-8397-449fe1ab9918"} {"msg":"Upstairs opts: Upstairs UUID: 6a3ca455-08b9-4dbe-8019-0366253bea7a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:12.588198431Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1333,"session_id":"b30686e4-2885-4734-8397-449fe1ab9918"} {"msg":"Crucible stats registered with UUID: 6a3ca455-08b9-4dbe-8019-0366253bea7a","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:12.588210489Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1333,"session_id":"b30686e4-2885-4734-8397-449fe1ab9918"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:55:12.588219717Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1333,"session_id":"b30686e4-2885-4734-8397-449fe1ab9918"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:12.588347828Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1333,"":"io task","client":"2","":"downstairs","session_id":"b30686e4-2885-4734-8397-449fe1ab9918"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:12.588364316Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1333,"":"io task","client":"0","":"downstairs","session_id":"b30686e4-2885-4734-8397-449fe1ab9918"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:12.588374655Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1333,"":"io task","client":"1","":"downstairs","session_id":"b30686e4-2885-4734-8397-449fe1ab9918"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:12.588536651Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1333,"task":"crutest"} {"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:12.588552288Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1333} {"msg":"6a3ca455-08b9-4dbe-8019-0366253bea7a active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:12.588583714Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1333,"session_id":"b30686e4-2885-4734-8397-449fe1ab9918"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:12.588593122Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1333,"client":"0","":"downstairs","session_id":"b30686e4-2885-4734-8397-449fe1ab9918"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:12.588602811Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1333,"client":"1","":"downstairs","session_id":"b30686e4-2885-4734-8397-449fe1ab9918"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:12.588612619Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1333,"client":"2","":"downstairs","session_id":"b30686e4-2885-4734-8397-449fe1ab9918"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:12.588622248Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1333,"":"io task","client":"2","":"downstairs","session_id":"b30686e4-2885-4734-8397-449fe1ab9918"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:12.588632346Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1333,"":"io task","client":"0","":"downstairs","session_id":"b30686e4-2885-4734-8397-449fe1ab9918"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:12.588642565Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1333,"":"io task","client":"1","":"downstairs","session_id":"b30686e4-2885-4734-8397-449fe1ab9918"} {"msg":"ds_connection connected from Ok(127.0.0.1:46668)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:12.588696407Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1333,"":"io task","client":"2","":"downstairs","session_id":"b30686e4-2885-4734-8397-449fe1ab9918"} {"msg":"ds_connection connected from Ok(127.0.0.1:46277)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:12.588839546Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1333,"":"io task","client":"0","":"downstairs","session_id":"b30686e4-2885-4734-8397-449fe1ab9918"} {"msg":"ds_connection connected from Ok(127.0.0.1:42673)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:12.588866932Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1333,"":"io task","client":"1","":"downstairs","session_id":"b30686e4-2885-4734-8397-449fe1ab9918"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:12.589349091Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1333,"client":"1","":"downstairs","session_id":"b30686e4-2885-4734-8397-449fe1ab9918"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:12.589399813Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1333,"client":"0","":"downstairs","session_id":"b30686e4-2885-4734-8397-449fe1ab9918"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:12.589413511Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1333,"client":"2","":"downstairs","session_id":"b30686e4-2885-4734-8397-449fe1ab9918"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:12.58969153Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1333,"session_id":"b30686e4-2885-4734-8397-449fe1ab9918"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:12.589740523Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1333,"session_id":"b30686e4-2885-4734-8397-449fe1ab9918"} {"msg":"[0]R flush_numbers[0..12]: [78, 83, 79, 88, 88, 83, 81, 95, 80, 95, 95, 78]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:12.589791975Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1333,"":"downstairs","":"downstairs","session_id":"b30686e4-2885-4734-8397-449fe1ab9918"} {"msg":"[0]R generation[0..12]: [43, 46, 44, 47, 47, 46, 45, 49, 44, 49, 49, 43]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:12.589803074Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1333,"":"downstairs","":"downstairs","session_id":"b30686e4-2885-4734-8397-449fe1ab9918"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:12.589814002Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1333,"":"downstairs","":"downstairs","session_id":"b30686e4-2885-4734-8397-449fe1ab9918"} {"msg":"[1]R flush_numbers[0..12]: [78, 83, 79, 88, 88, 83, 81, 95, 80, 95, 95, 78]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:12.589824421Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1333,"":"downstairs","":"downstairs","session_id":"b30686e4-2885-4734-8397-449fe1ab9918"} {"msg":"[1]R generation[0..12]: [43, 46, 44, 47, 47, 46, 45, 49, 44, 49, 49, 43]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:12.589834739Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1333,"":"downstairs","":"downstairs","session_id":"b30686e4-2885-4734-8397-449fe1ab9918"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:12.589845787Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1333,"":"downstairs","":"downstairs","session_id":"b30686e4-2885-4734-8397-449fe1ab9918"} {"msg":"[2]R flush_numbers[0..12]: [78, 83, 79, 88, 88, 83, 81, 95, 80, 95, 95, 78]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:12.589855996Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1333,"":"downstairs","":"downstairs","session_id":"b30686e4-2885-4734-8397-449fe1ab9918"} {"msg":"[2]R generation[0..12]: [43, 46, 44, 47, 47, 46, 45, 49, 44, 49, 49, 43]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:12.589866314Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1333,"":"downstairs","":"downstairs","session_id":"b30686e4-2885-4734-8397-449fe1ab9918"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:12.589879692Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1333,"":"downstairs","":"downstairs","session_id":"b30686e4-2885-4734-8397-449fe1ab9918"} {"msg":"Max found gen is 50","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:12.589889371Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1333,"":"downstairs","":"downstairs","session_id":"b30686e4-2885-4734-8397-449fe1ab9918"} {"msg":"Generation requested: 50 >= found:50","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:12.58989899Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1333,"":"downstairs","":"downstairs","session_id":"b30686e4-2885-4734-8397-449fe1ab9918"} {"msg":"Next flush: 96","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:12.589908448Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1333,"":"downstairs","":"downstairs","session_id":"b30686e4-2885-4734-8397-449fe1ab9918"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:12.589917897Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1333,"":"downstairs","":"downstairs","session_id":"b30686e4-2885-4734-8397-449fe1ab9918"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:12.589927685Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1333,"session_id":"b30686e4-2885-4734-8397-449fe1ab9918"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:12.589936564Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1333,"client":"0","":"downstairs","session_id":"b30686e4-2885-4734-8397-449fe1ab9918"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:12.589946323Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1333,"client":"1","":"downstairs","session_id":"b30686e4-2885-4734-8397-449fe1ab9918"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:12.589955961Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1333,"client":"2","":"downstairs","session_id":"b30686e4-2885-4734-8397-449fe1ab9918"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:12.58996581Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1333,"session_id":"b30686e4-2885-4734-8397-449fe1ab9918"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:12.589974908Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1333,"session_id":"b30686e4-2885-4734-8397-449fe1ab9918"} {"msg":"6a3ca455-08b9-4dbe-8019-0366253bea7a is now active with session: b30686e4-2885-4734-8397-449fe1ab9918","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:12.589984777Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1333,"session_id":"b30686e4-2885-4734-8397-449fe1ab9918"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:12.589993496Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1333,"session_id":"b30686e4-2885-4734-8397-449fe1ab9918"} {"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:12.590002694Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1333} {"msg":"Activated sub_volume 6a3ca455-08b9-4dbe-8019-0366253bea7a","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:12.590010863Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1333} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:12.590018902Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1333,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:12.590351423Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1333,"task":"crutest"} 01/30 Write block 250 len 32768 data: 4 3 3 3 3 3 3 4 02/30 Write block 289 len 4096 data: 4 03/30 Read block 389 len 36864 04/30 Read block 538 len 8192 05/30 Read block 280 len 24576 06/30 Write block 184 len 16384 data: 4 2 2 3 07/30 Read block 115 len 40960 08/30 Flush 09/30 Write block 466 len 24576 data: 6 6 7 6 6 8 10/30 Read block 167 len 36864 11/30 Read block 479 len 16384 12/30 Read block 153 len 24576 13/30 Read block 109 len 20480 14/30 Read block 255 len 4096 15/30 Write block 226 len 16384 data: 3 3 4 3 16/30 Read block 430 len 4096 17/30 Read block 503 len 32768 18/30 Write block 114 len 32768 data: 6 6 5 5 4 4 3 3 19/30 Read block 547 len 20480 20/30 Read block 475 len 32768 21/30 Read block 557 len 20480 22/30 Read block 368 len 8192 23/30 Read block 205 len 16384 24/30 Write block 263 len 4096 data: 7 25/30 Read block 136 len 12288 26/30 Flush 27/30 Write block 265 len 28672 data: 7 9 9 6 6 6 7 28/30 Write block 211 len 28672 data: 4 5 5 5 4 3 3 29/30 Read block 244 len 40960 30/30 Read block 420 len 28672 ---------------------------------------------------------------- Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:22 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1008 Acked Write 6 Done Done Done false 1009 Acked Read 9 Sent Done Done false 1010 Acked Read 4 Sent Done Done false 1011 Acked Read 6 Sent Done Done false 1012 Acked Read 5 Sent Done Done false 1013 Acked Read 1 Sent Done Done false 1014 Acked Write 4 Sent Done Done false 1015 Acked Read 1 Sent Done Done false 1016 Acked Read 8 Sent Done Done false 1017 Acked Write 8 Sent Done Done false 1018 Acked Read 5 Sent Done Done false 1019 Acked Read 8 Sent Done Done false 1020 Acked Read 5 Sent Done Done false 1021 Acked Read 2 Sent Done Done false 1022 Acked Read 4 Sent Done Done false 1023 Acked Write 1 Sent Done Done false 1024 Acked Read 3 Sent Done Done false 1025 Acked Flush 0 Sent Done Done false 1026 Acked Write 7 Sent Done Done false 1027 Acked Write 7 Sent Done Done false 1028 Acked Read 10 Sent Done Done false 1029 Acked Read 7 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 21 1 0 22 Done 1 21 22 44 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1007)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:55:12.599609967Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1333} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:12.599785701Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1333,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 43 43 43 78 78 78 F F F 1 020-039 46 46 46 83 83 83 F F F 2 040-059 44 44 44 79 79 79 F F F 3 060-079 47 47 47 88 88 88 F F F 4 080-099 47 47 47 88 88 88 F F F 5 100-119 46 50 50 83 97 97 F F F <--- 6 120-139 45 50 50 81 97 97 F F F <--- 7 140-159 49 49 49 95 95 95 F F F 8 160-179 44 44 44 80 80 80 F F F 9 180-199 50 50 50 96 96 96 F F F 10 200-219 49 49 49 95 95 95 F T T <--- 11 220-239 43 50 50 78 97 97 F F F <--- 12 240-259 50 50 50 96 96 96 F F F 13 260-279 48 50 50 92 97 97 F T T <--- 14 280-299 50 50 50 96 96 96 F F F 15 300-319 49 49 49 95 95 95 F F F 16 320-339 49 49 49 95 95 95 F F F 17 340-359 46 46 46 83 83 83 F F F 18 360-379 43 43 43 78 78 78 F F F 19 380-399 44 44 44 79 79 79 F F F 20 400-419 48 48 48 93 93 93 F F F 21 420-439 48 48 48 94 94 94 F F F 22 440-459 39 39 39 72 72 72 F F F 23 460-479 48 50 50 92 97 97 T F F <--- 24 480-499 41 41 41 76 76 76 F F F 25 500-519 48 48 48 93 93 93 F F F 26 520-539 48 48 48 92 92 92 F F F 27 540-559 45 45 45 81 81 81 F F F 28 560-579 48 48 48 94 94 94 F F F 29 580-599 38 38 38 70 70 70 F F F Max gen: 50, Max flush: 97 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 25 Verifying data now No change after verify Loop: 25 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 43 43 43 78 78 78 F F F 1 020-039 46 46 46 83 83 83 F F F 2 040-059 44 44 44 79 79 79 F F F 3 060-079 47 47 47 88 88 88 F F F 4 080-099 47 47 47 88 88 88 F F F 5 100-119 50 50 50 97 97 97 F F F 6 120-139 50 50 50 97 97 97 F F F 7 140-159 49 49 49 95 95 95 F F F 8 160-179 44 44 44 80 80 80 F F F 9 180-199 50 50 50 96 96 96 F F F 10 200-219 51 51 51 98 98 98 F F F 11 220-239 50 50 50 97 97 97 F F F 12 240-259 50 50 50 96 96 96 F F F 13 260-279 51 51 51 98 98 98 F F F 14 280-299 50 50 50 96 96 96 F F F 15 300-319 49 49 49 95 95 95 F F F 16 320-339 49 49 49 95 95 95 F F F 17 340-359 46 46 46 83 83 83 F F F 18 360-379 43 43 43 78 78 78 F F F 19 380-399 44 44 44 79 79 79 F F F 20 400-419 48 48 48 93 93 93 F F F 21 420-439 48 48 48 94 94 94 F F F 22 440-459 39 39 39 72 72 72 F F F 23 460-479 50 50 50 97 97 97 F F F 24 480-499 41 41 41 76 76 76 F F F 25 500-519 48 48 48 93 93 93 F F F 26 520-539 48 48 48 92 92 92 F F F 27 540-559 45 45 45 81 81 81 F F F 28 560-579 48 48 48 94 94 94 F F F 29 580-599 38 38 38 70 70 70 F F F Max gen: 51, Max flush: 98 Begin loop 26 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:16.8410048Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1341,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:55:16.848099383Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1341,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:16.848154855Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1341,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:16.854903479Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1341,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:16.85517083Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1341} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:16.855193766Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1341} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:16.855202455Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1341} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:16.855213993Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1341} {"msg":"Crucible 4d167c03-65aa-4d8f-9570-723417b5fca0 has session id: 55346cd8-f54b-409e-a8db-288d30c46669","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:16.855531186Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1341,"session_id":"55346cd8-f54b-409e-a8db-288d30c46669"} {"msg":"Upstairs opts: Upstairs UUID: 4d167c03-65aa-4d8f-9570-723417b5fca0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:16.855553413Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1341,"session_id":"55346cd8-f54b-409e-a8db-288d30c46669"} {"msg":"Crucible stats registered with UUID: 4d167c03-65aa-4d8f-9570-723417b5fca0","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:16.855563342Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1341,"session_id":"55346cd8-f54b-409e-a8db-288d30c46669"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:55:16.85557255Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1341,"session_id":"55346cd8-f54b-409e-a8db-288d30c46669"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:16.85571082Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1341,"":"io task","client":"2","":"downstairs","session_id":"55346cd8-f54b-409e-a8db-288d30c46669"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:16.855727388Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1341,"":"io task","client":"0","":"downstairs","session_id":"55346cd8-f54b-409e-a8db-288d30c46669"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:16.855737656Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1341,"":"io task","client":"1","":"downstairs","session_id":"55346cd8-f54b-409e-a8db-288d30c46669"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:16.855887114Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1341,"task":"crutest"} {"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:16.855898212Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1341} {"msg":"4d167c03-65aa-4d8f-9570-723417b5fca0 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:16.8559158Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1341,"session_id":"55346cd8-f54b-409e-a8db-288d30c46669"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:16.855924808Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1341,"client":"0","":"downstairs","session_id":"55346cd8-f54b-409e-a8db-288d30c46669"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:16.855934557Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1341,"client":"1","":"downstairs","session_id":"55346cd8-f54b-409e-a8db-288d30c46669"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:16.855944226Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1341,"client":"2","":"downstairs","session_id":"55346cd8-f54b-409e-a8db-288d30c46669"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:16.855953764Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1341,"":"io task","client":"2","":"downstairs","session_id":"55346cd8-f54b-409e-a8db-288d30c46669"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:16.855963903Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1341,"":"io task","client":"0","":"downstairs","session_id":"55346cd8-f54b-409e-a8db-288d30c46669"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:16.855974011Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1341,"":"io task","client":"1","":"downstairs","session_id":"55346cd8-f54b-409e-a8db-288d30c46669"} {"msg":"ds_connection connected from Ok(127.0.0.1:50487)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:16.856070397Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1341,"":"io task","client":"1","":"downstairs","session_id":"55346cd8-f54b-409e-a8db-288d30c46669"} {"msg":"ds_connection connected from Ok(127.0.0.1:41650)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:16.856095653Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1341,"":"io task","client":"2","":"downstairs","session_id":"55346cd8-f54b-409e-a8db-288d30c46669"} {"msg":"ds_connection connected from Ok(127.0.0.1:33821)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:16.856205117Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1341,"":"io task","client":"0","":"downstairs","session_id":"55346cd8-f54b-409e-a8db-288d30c46669"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:16.85679822Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1341,"client":"1","":"downstairs","session_id":"55346cd8-f54b-409e-a8db-288d30c46669"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:16.856824016Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1341,"client":"2","":"downstairs","session_id":"55346cd8-f54b-409e-a8db-288d30c46669"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:16.856835974Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1341,"client":"0","":"downstairs","session_id":"55346cd8-f54b-409e-a8db-288d30c46669"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:16.857152977Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1341,"session_id":"55346cd8-f54b-409e-a8db-288d30c46669"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:16.85720229Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1341,"session_id":"55346cd8-f54b-409e-a8db-288d30c46669"} {"msg":"[0]R flush_numbers[0..12]: [78, 83, 79, 88, 88, 97, 97, 95, 80, 96, 98, 97]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:16.857241094Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1341,"":"downstairs","":"downstairs","session_id":"55346cd8-f54b-409e-a8db-288d30c46669"} {"msg":"[0]R generation[0..12]: [43, 46, 44, 47, 47, 50, 50, 49, 44, 50, 51, 50]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:16.857252383Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1341,"":"downstairs","":"downstairs","session_id":"55346cd8-f54b-409e-a8db-288d30c46669"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:16.857263411Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1341,"":"downstairs","":"downstairs","session_id":"55346cd8-f54b-409e-a8db-288d30c46669"} {"msg":"[1]R flush_numbers[0..12]: [78, 83, 79, 88, 88, 97, 97, 95, 80, 96, 98, 97]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:16.857274099Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1341,"":"downstairs","":"downstairs","session_id":"55346cd8-f54b-409e-a8db-288d30c46669"} {"msg":"[1]R generation[0..12]: [43, 46, 44, 47, 47, 50, 50, 49, 44, 50, 51, 50]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:16.857284448Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1341,"":"downstairs","":"downstairs","session_id":"55346cd8-f54b-409e-a8db-288d30c46669"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:16.857295306Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1341,"":"downstairs","":"downstairs","session_id":"55346cd8-f54b-409e-a8db-288d30c46669"} {"msg":"[2]R flush_numbers[0..12]: [78, 83, 79, 88, 88, 97, 97, 95, 80, 96, 98, 97]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:16.857305895Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1341,"":"downstairs","":"downstairs","session_id":"55346cd8-f54b-409e-a8db-288d30c46669"} {"msg":"[2]R generation[0..12]: [43, 46, 44, 47, 47, 50, 50, 49, 44, 50, 51, 50]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:16.857316303Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1341,"":"downstairs","":"downstairs","session_id":"55346cd8-f54b-409e-a8db-288d30c46669"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:16.857327361Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1341,"":"downstairs","":"downstairs","session_id":"55346cd8-f54b-409e-a8db-288d30c46669"} {"msg":"Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:16.85733676Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1341,"":"downstairs","":"downstairs","session_id":"55346cd8-f54b-409e-a8db-288d30c46669"} {"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:16.857346529Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1341,"":"downstairs","":"downstairs","session_id":"55346cd8-f54b-409e-a8db-288d30c46669"} {"msg":"Next flush: 99","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:16.857355927Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1341,"":"downstairs","":"downstairs","session_id":"55346cd8-f54b-409e-a8db-288d30c46669"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:16.857365296Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1341,"":"downstairs","":"downstairs","session_id":"55346cd8-f54b-409e-a8db-288d30c46669"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:16.857375274Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1341,"session_id":"55346cd8-f54b-409e-a8db-288d30c46669"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:16.857384093Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1341,"client":"0","":"downstairs","session_id":"55346cd8-f54b-409e-a8db-288d30c46669"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:16.857395671Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1341,"client":"1","":"downstairs","session_id":"55346cd8-f54b-409e-a8db-288d30c46669"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:16.8574056Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1341,"client":"2","":"downstairs","session_id":"55346cd8-f54b-409e-a8db-288d30c46669"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:16.857415638Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1341,"session_id":"55346cd8-f54b-409e-a8db-288d30c46669"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:16.857424707Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1341,"session_id":"55346cd8-f54b-409e-a8db-288d30c46669"} {"msg":"4d167c03-65aa-4d8f-9570-723417b5fca0 is now active with session: 55346cd8-f54b-409e-a8db-288d30c46669","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:16.857434796Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1341,"session_id":"55346cd8-f54b-409e-a8db-288d30c46669"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:16.857443584Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1341,"session_id":"55346cd8-f54b-409e-a8db-288d30c46669"} {"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:16.857452663Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1341} {"msg":"Activated sub_volume 4d167c03-65aa-4d8f-9570-723417b5fca0","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:16.857460552Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1341} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:16.857468601Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1341,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:16.857774675Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1341,"task":"crutest"} 01/30 Write block 358 len 16384 data: 6 6 6 5 02/30 Read block 470 len 32768 03/30 Write block 407 len 32768 data: 6 5 7 10 13 11 11 8 04/30 Write block 282 len 4096 data: 4 05/30 Read block 465 len 12288 06/30 Flush 07/30 Write block 181 len 4096 data: 3 08/30 Write block 589 len 32768 data: 5 4 7 8 8 7 6 4 09/30 Flush 10/30 Write block 38 len 20480 data: 10 10 9 9 8 11/30 Read block 29 len 28672 12/30 Write block 427 len 16384 data: 6 6 6 5 13/30 Read block 99 len 40960 14/30 Read block 518 len 32768 15/30 Write block 456 len 24576 data: 5 5 5 5 3 3 16/30 Write block 243 len 16384 data: 6 8 9 9 17/30 Flush 18/30 Write block 419 len 28672 data: 3 3 3 5 5 5 6 19/30 Write block 166 len 12288 data: 5 5 3 20/30 Read block 436 len 8192 21/30 Read block 116 len 16384 22/30 Read block 61 len 32768 23/30 Read block 30 len 4096 24/30 Write block 111 len 32768 data: 6 6 6 7 7 6 6 5 25/30 Write block 20 len 40960 data: 11 10 11 8 9 9 8 6 6 6 26/30 Write block 92 len 28672 data: 5 5 4 5 5 4 5 27/30 Write block 531 len 24576 data: 4 5 6 5 5 5 28/30 Read block 231 len 12288 29/30 Write block 270 len 20480 data: 7 8 7 5 6 30/30 Write block 208 len 8192 data: 7 5 ---------------------------------------------------------------- Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Read 8 Sent Done Done false 1002 Acked Write 8 Sent Done Done false 1003 Acked Write 1 Sent Done Done false 1004 Acked Read 3 Sent Done Done false 1005 Acked Flush 0 Sent Done Done false 1006 Acked Write 1 Sent Done Done false 1007 Acked Write 8 Sent Done Done false 1008 Acked Flush 0 Sent Done Done false 1009 Acked Write 5 Sent Done Done false 1010 Acked Read 7 Sent Done Done false 1011 Acked Write 4 Sent Done Done false 1012 Acked Read 10 Sent Done Done false 1013 Acked Read 8 Sent Done Done false 1014 Acked Write 6 Sent Done Done false 1015 Acked Write 4 Sent Done Done false 1016 Acked Flush 0 Sent Done Done false 1017 Acked Write 7 Sent Done Done false 1018 Acked Write 3 Sent Done Done false 1019 Acked Read 2 Sent Done Done false 1020 Acked Read 4 Sent Done Done false 1021 Acked Read 8 Sent Done Done false 1022 Acked Read 1 Sent Done Done false 1023 Acked Write 8 Sent Done Done false 1024 Acked Write 10 Sent Done Done false 1025 Acked Write 7 Sent Done Done false 1026 Acked Write 6 Sent Done Done false 1027 Acked Read 3 Sent Done Done false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 2 2 33 Done 1 28 28 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:55:16.866495429Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1341} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:16.866678362Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1341,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 43 43 43 78 78 78 F F F 1 020-039 46 52 52 83 101 101 F T T <--- 2 040-059 44 52 52 79 101 101 F F F <--- 3 060-079 47 47 47 88 88 88 F F F 4 080-099 47 47 47 88 88 88 F T T <--- 5 100-119 50 50 50 97 97 97 F T T <--- 6 120-139 50 50 50 97 97 97 F F F 7 140-159 49 49 49 95 95 95 F F F 8 160-179 44 44 44 80 80 80 F T T <--- 9 180-199 50 52 52 96 100 100 F F F <--- 10 200-219 51 51 51 98 98 98 F T T <--- 11 220-239 50 50 50 97 97 97 F F F 12 240-259 50 52 52 96 101 101 F F F <--- 13 260-279 51 51 51 98 98 98 F T T <--- 14 280-299 50 52 52 96 99 99 F F F <--- 15 300-319 49 49 49 95 95 95 F F F 16 320-339 49 49 49 95 95 95 F F F 17 340-359 46 52 52 83 99 99 T F F <--- 18 360-379 43 52 52 78 99 99 T F F <--- 19 380-399 44 44 44 79 79 79 F F F 20 400-419 48 52 52 93 99 99 F T T <--- 21 420-439 48 52 52 94 101 101 F T T <--- 22 440-459 39 52 52 72 101 101 F F F <--- 23 460-479 50 52 52 97 101 101 F F F <--- 24 480-499 41 41 41 76 76 76 F F F 25 500-519 48 48 48 93 93 93 F F F 26 520-539 48 48 48 92 92 92 F T T <--- 27 540-559 45 45 45 81 81 81 F F F 28 560-579 48 48 48 94 94 94 F F F 29 580-599 38 52 52 70 100 100 F F F <--- Max gen: 52, Max flush: 101 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 26 Verifying data now No change after verify Loop: 26 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 43 43 43 78 78 78 F F F 1 020-039 53 53 53 102 102 102 F F F 2 040-059 52 52 52 101 101 101 F F F 3 060-079 47 47 47 88 88 88 F F F 4 080-099 53 53 53 102 102 102 F F F 5 100-119 53 53 53 102 102 102 F F F 6 120-139 50 50 50 97 97 97 F F F 7 140-159 49 49 49 95 95 95 F F F 8 160-179 53 53 53 102 102 102 F F F 9 180-199 52 52 52 100 100 100 F F F 10 200-219 53 53 53 102 102 102 F F F 11 220-239 50 50 50 97 97 97 F F F 12 240-259 52 52 52 101 101 101 F F F 13 260-279 53 53 53 102 102 102 F F F 14 280-299 52 52 52 99 99 99 F F F 15 300-319 49 49 49 95 95 95 F F F 16 320-339 49 49 49 95 95 95 F F F 17 340-359 52 52 52 99 99 99 F F F 18 360-379 52 52 52 99 99 99 F F F 19 380-399 44 44 44 79 79 79 F F F 20 400-419 53 53 53 102 102 102 F F F 21 420-439 53 53 53 102 102 102 F F F 22 440-459 52 52 52 101 101 101 F F F 23 460-479 52 52 52 101 101 101 F F F 24 480-499 41 41 41 76 76 76 F F F 25 500-519 48 48 48 93 93 93 F F F 26 520-539 53 53 53 102 102 102 F F F 27 540-559 45 45 45 81 81 81 F F F 28 560-579 48 48 48 94 94 94 F F F 29 580-599 52 52 52 100 100 100 F F F Max gen: 53, Max flush: 102 Begin loop 27 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:21.133664399Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1349,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:21.142094405Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1349,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:21.142388202Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1349} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:21.142413768Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1349} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:21.142422397Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1349} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:21.142433895Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1349} {"msg":"Crucible a7fe6fb6-653c-4915-a63b-dd1ad13460f2 has session id: b49b1df9-86c6-4419-9d1c-ddc033aa68de","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:21.142778914Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1349,"session_id":"b49b1df9-86c6-4419-9d1c-ddc033aa68de"} {"msg":"Upstairs opts: Upstairs UUID: a7fe6fb6-653c-4915-a63b-dd1ad13460f2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:21.142800311Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1349,"session_id":"b49b1df9-86c6-4419-9d1c-ddc033aa68de"} {"msg":"Crucible stats registered with UUID: a7fe6fb6-653c-4915-a63b-dd1ad13460f2","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:21.14281026Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1349,"session_id":"b49b1df9-86c6-4419-9d1c-ddc033aa68de"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:55:21.142819688Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1349,"session_id":"b49b1df9-86c6-4419-9d1c-ddc033aa68de"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:21.142964557Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1349,"":"io task","client":"1","":"downstairs","session_id":"b49b1df9-86c6-4419-9d1c-ddc033aa68de"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:21.142995573Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1349,"":"io task","client":"0","":"downstairs","session_id":"b49b1df9-86c6-4419-9d1c-ddc033aa68de"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:21.143006541Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1349,"":"io task","client":"2","":"downstairs","session_id":"b49b1df9-86c6-4419-9d1c-ddc033aa68de"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:21.14314871Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1349,"task":"crutest"} {"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:21.143158379Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1349} {"msg":"a7fe6fb6-653c-4915-a63b-dd1ad13460f2 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:21.143174516Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1349,"session_id":"b49b1df9-86c6-4419-9d1c-ddc033aa68de"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:21.143183535Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1349,"client":"0","":"downstairs","session_id":"b49b1df9-86c6-4419-9d1c-ddc033aa68de"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:21.143193283Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1349,"client":"1","":"downstairs","session_id":"b49b1df9-86c6-4419-9d1c-ddc033aa68de"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:21.143203212Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1349,"client":"2","":"downstairs","session_id":"b49b1df9-86c6-4419-9d1c-ddc033aa68de"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:21.14321282Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1349,"":"io task","client":"2","":"downstairs","session_id":"b49b1df9-86c6-4419-9d1c-ddc033aa68de"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:21.143222929Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1349,"":"io task","client":"0","":"downstairs","session_id":"b49b1df9-86c6-4419-9d1c-ddc033aa68de"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:21.143235307Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1349,"":"io task","client":"1","":"downstairs","session_id":"b49b1df9-86c6-4419-9d1c-ddc033aa68de"} {"msg":"ds_connection connected from Ok(127.0.0.1:34957)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:21.143289879Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1349,"":"io task","client":"2","":"downstairs","session_id":"b49b1df9-86c6-4419-9d1c-ddc033aa68de"} {"msg":"ds_connection connected from Ok(127.0.0.1:61684)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:21.143302987Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1349,"":"io task","client":"1","":"downstairs","session_id":"b49b1df9-86c6-4419-9d1c-ddc033aa68de"} {"msg":"ds_connection connected from Ok(127.0.0.1:56133)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:21.143313846Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1349,"":"io task","client":"0","":"downstairs","session_id":"b49b1df9-86c6-4419-9d1c-ddc033aa68de"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:21.143946662Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1349,"client":"1","":"downstairs","session_id":"b49b1df9-86c6-4419-9d1c-ddc033aa68de"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:21.143972568Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1349,"client":"0","":"downstairs","session_id":"b49b1df9-86c6-4419-9d1c-ddc033aa68de"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:21.14409964Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1349,"client":"2","":"downstairs","session_id":"b49b1df9-86c6-4419-9d1c-ddc033aa68de"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:21.144240379Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1349,"session_id":"b49b1df9-86c6-4419-9d1c-ddc033aa68de"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:21.144259956Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1349,"session_id":"b49b1df9-86c6-4419-9d1c-ddc033aa68de"} {"msg":"[0]R flush_numbers[0..12]: [78, 102, 101, 88, 102, 102, 97, 95, 102, 100, 102, 97]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:21.14443581Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1349,"":"downstairs","":"downstairs","session_id":"b49b1df9-86c6-4419-9d1c-ddc033aa68de"} {"msg":"[0]R generation[0..12]: [43, 53, 52, 47, 53, 53, 50, 49, 53, 52, 53, 50]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:21.144459457Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1349,"":"downstairs","":"downstairs","session_id":"b49b1df9-86c6-4419-9d1c-ddc033aa68de"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:21.144471505Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1349,"":"downstairs","":"downstairs","session_id":"b49b1df9-86c6-4419-9d1c-ddc033aa68de"} {"msg":"[1]R flush_numbers[0..12]: [78, 102, 101, 88, 102, 102, 97, 95, 102, 100, 102, 97]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:21.144482243Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1349,"":"downstairs","":"downstairs","session_id":"b49b1df9-86c6-4419-9d1c-ddc033aa68de"} {"msg":"[1]R generation[0..12]: [43, 53, 52, 47, 53, 53, 50, 49, 53, 52, 53, 50]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:21.144492792Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1349,"":"downstairs","":"downstairs","session_id":"b49b1df9-86c6-4419-9d1c-ddc033aa68de"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:21.14450682Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1349,"":"downstairs","":"downstairs","session_id":"b49b1df9-86c6-4419-9d1c-ddc033aa68de"} {"msg":"[2]R flush_numbers[0..12]: [78, 102, 101, 88, 102, 102, 97, 95, 102, 100, 102, 97]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:21.144517678Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1349,"":"downstairs","":"downstairs","session_id":"b49b1df9-86c6-4419-9d1c-ddc033aa68de"} {"msg":"[2]R generation[0..12]: [43, 53, 52, 47, 53, 53, 50, 49, 53, 52, 53, 50]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:21.144528226Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1349,"":"downstairs","":"downstairs","session_id":"b49b1df9-86c6-4419-9d1c-ddc033aa68de"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:21.144539255Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1349,"":"downstairs","":"downstairs","session_id":"b49b1df9-86c6-4419-9d1c-ddc033aa68de"} {"msg":"Max found gen is 54","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:21.144548783Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1349,"":"downstairs","":"downstairs","session_id":"b49b1df9-86c6-4419-9d1c-ddc033aa68de"} {"msg":"Generation requested: 54 >= found:54","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:21.144558812Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1349,"":"downstairs","":"downstairs","session_id":"b49b1df9-86c6-4419-9d1c-ddc033aa68de"} {"msg":"Next flush: 103","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:21.14456843Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1349,"":"downstairs","":"downstairs","session_id":"b49b1df9-86c6-4419-9d1c-ddc033aa68de"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:21.144578139Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1349,"":"downstairs","":"downstairs","session_id":"b49b1df9-86c6-4419-9d1c-ddc033aa68de"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:21.144588647Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1349,"session_id":"b49b1df9-86c6-4419-9d1c-ddc033aa68de"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:21.144597656Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1349,"client":"0","":"downstairs","session_id":"b49b1df9-86c6-4419-9d1c-ddc033aa68de"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:21.144607775Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1349,"client":"1","":"downstairs","session_id":"b49b1df9-86c6-4419-9d1c-ddc033aa68de"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:21.144617733Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1349,"client":"2","":"downstairs","session_id":"b49b1df9-86c6-4419-9d1c-ddc033aa68de"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:21.144627802Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1349,"session_id":"b49b1df9-86c6-4419-9d1c-ddc033aa68de"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:21.14463703Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1349,"session_id":"b49b1df9-86c6-4419-9d1c-ddc033aa68de"} {"msg":"a7fe6fb6-653c-4915-a63b-dd1ad13460f2 is now active with session: b49b1df9-86c6-4419-9d1c-ddc033aa68de","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:21.144647019Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1349,"session_id":"b49b1df9-86c6-4419-9d1c-ddc033aa68de"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:21.144657257Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1349,"session_id":"b49b1df9-86c6-4419-9d1c-ddc033aa68de"} {"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:21.144666416Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1349} {"msg":"Activated sub_volume a7fe6fb6-653c-4915-a63b-dd1ad13460f2","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:21.144674425Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1349} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:21.144682704Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1349,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:21.144988109Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1349,"task":"crutest"} 01/30 Write block 382 len 28672 data: 2 2 2 3 4 4 4 02/30 Write block 523 len 8192 data: 3 3 03/30 Read block 569 len 20480 04/30 Read block 567 len 28672 05/30 Read block 552 len 36864 06/30 Write block 559 len 32768 data: 4 4 6 6 6 7 6 4 07/30 Read block 383 len 8192 08/30 Write block 109 len 12288 data: 3 4 7 09/30 Flush 10/30 Write block 376 len 16384 data: 3 3 2 2 11/30 Read block 253 len 12288 12/30 Read block 1 len 16384 13/30 Write block 554 len 32768 data: 5 5 5 6 6 5 5 7 14/30 Read block 552 len 12288 15/30 Read block 395 len 12288 16/30 Write block 176 len 8192 data: 6 5 17/30 Read block 273 len 12288 18/30 Read block 361 len 20480 19/30 Write block 432 len 12288 data: 6 5 5 20/30 Read block 449 len 28672 21/30 Read block 186 len 24576 22/30 Write block 354 len 28672 data: 6 6 7 7 7 7 7 23/30 Write block 330 len 36864 data: 6 6 5 4 4 5 5 5 5 24/30 Write block 413 len 4096 data: 12 25/30 Read block 92 len 12288 26/30 Write block 255 len 16384 data: 4 4 5 6 27/30 Write block 370 len 24576 data: 3 4 5 4 4 4 28/30 Read block 3 len 24576 29/30 Read block 92 len 4096 30/30 Write block 464 len 12288 data: 5 5 7 ---------------------------------------------------------------- Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Sent false 1001 Acked Write 2 Done Done Sent false 1002 Acked Read 5 Done Done Sent false 1003 Acked Read 7 Done Done Sent false 1004 Acked Read 9 Done Done Sent false 1005 Acked Write 8 Done Done Sent false 1006 Acked Read 2 Done Done Sent false 1007 Acked Write 3 Done Done Sent false 1008 Acked Flush 0 Done Done Sent false 1009 Acked Write 4 Done Done Sent false 1010 Acked Read 3 Done Done Sent false 1011 Acked Read 4 Done Done Sent false 1012 Acked Write 8 Done Done Sent false 1013 Acked Read 3 Done Done Sent false 1014 Acked Read 3 Done Done Sent false 1015 Acked Write 2 Done Done Sent false 1016 Acked Read 3 Done Done Sent false 1017 Acked Read 5 Done Done Sent false 1018 Acked Write 3 Done Done Sent false 1019 Acked Read 7 Done Done Sent false 1020 Acked Read 6 Done Done Sent false 1021 Acked Write 7 Done Done Sent false 1022 Acked Write 9 Done Done Sent false 1023 Acked Write 1 Done Done Sent false 1024 Acked Read 3 Done Done Sent false 1025 Acked Write 4 Done Done Sent false 1026 Acked Write 6 Done Done Sent false 1027 Acked Read 6 Done Done Sent false 1028 Acked Read 1 Done Done Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 30 32 Done 29 29 0 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1008)) Some(JobId(1008)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:55:21.153371372Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1349} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:21.153557654Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1349,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 43 43 43 78 78 78 F F F 1 020-039 53 53 53 102 102 102 F F F 2 040-059 52 52 52 101 101 101 F F F 3 060-079 47 47 47 88 88 88 F F F 4 080-099 53 53 53 102 102 102 F F F 5 100-119 54 54 53 103 103 102 F F F <--- 6 120-139 50 50 50 97 97 97 F F F 7 140-159 49 49 49 95 95 95 F F F 8 160-179 53 53 53 102 102 102 T T F <--- 9 180-199 52 52 52 100 100 100 F F F 10 200-219 53 53 53 102 102 102 F F F 11 220-239 50 50 50 97 97 97 F F F 12 240-259 52 52 52 101 101 101 T T F <--- 13 260-279 53 53 53 102 102 102 F F F 14 280-299 52 52 52 99 99 99 F F F 15 300-319 49 49 49 95 95 95 F F F 16 320-339 49 49 49 95 95 95 T T F <--- 17 340-359 52 52 52 99 99 99 T T F <--- 18 360-379 52 52 52 99 99 99 T T F <--- 19 380-399 54 54 44 103 103 79 F F F <--- 20 400-419 53 53 53 102 102 102 T T F <--- 21 420-439 53 53 53 102 102 102 T T F <--- 22 440-459 52 52 52 101 101 101 F F F 23 460-479 52 52 52 101 101 101 T T F <--- 24 480-499 41 41 41 76 76 76 F F F 25 500-519 48 48 48 93 93 93 F F F 26 520-539 54 54 53 103 103 102 F F F <--- 27 540-559 54 54 45 103 103 81 T T F <--- 28 560-579 54 54 48 103 103 94 T T F <--- 29 580-599 52 52 52 100 100 100 F F F Max gen: 54, Max flush: 103 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 27 Verifying data now No change after verify Loop: 27 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 43 43 43 78 78 78 F F F 1 020-039 53 53 53 102 102 102 F F F 2 040-059 52 52 52 101 101 101 F F F 3 060-079 47 47 47 88 88 88 F F F 4 080-099 53 53 53 102 102 102 F F F 5 100-119 54 54 54 103 103 103 F F F 6 120-139 50 50 50 97 97 97 F F F 7 140-159 49 49 49 95 95 95 F F F 8 160-179 55 55 55 104 104 104 F F F 9 180-199 52 52 52 100 100 100 F F F 10 200-219 53 53 53 102 102 102 F F F 11 220-239 50 50 50 97 97 97 F F F 12 240-259 55 55 55 104 104 104 F F F 13 260-279 53 53 53 102 102 102 F F F 14 280-299 52 52 52 99 99 99 F F F 15 300-319 49 49 49 95 95 95 F F F 16 320-339 55 55 55 104 104 104 F F F 17 340-359 55 55 55 104 104 104 F F F 18 360-379 55 55 55 104 104 104 F F F 19 380-399 54 54 54 103 103 103 F F F 20 400-419 55 55 55 104 104 104 F F F 21 420-439 55 55 55 104 104 104 F F F 22 440-459 52 52 52 101 101 101 F F F 23 460-479 55 55 55 104 104 104 F F F 24 480-499 41 41 41 76 76 76 F F F 25 500-519 48 48 48 93 93 93 F F F 26 520-539 54 54 54 103 103 103 F F F 27 540-559 55 55 55 104 104 104 F F F 28 560-579 55 55 55 104 104 104 F F F 29 580-599 52 52 52 100 100 100 F F F Max gen: 55, Max flush: 104 Begin loop 28 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:25.402673998Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1357,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:25.410845053Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1357,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:25.411144869Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1357} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:25.411173284Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1357} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:25.411182203Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1357} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:25.411193971Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1357} {"msg":"Crucible 89564478-016a-4da2-9b5e-d151e4542edc has session id: dbb96ce3-2469-4b96-8b5b-c469fce4458e","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:25.411516584Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1357,"session_id":"dbb96ce3-2469-4b96-8b5b-c469fce4458e"} {"msg":"Upstairs opts: Upstairs UUID: 89564478-016a-4da2-9b5e-d151e4542edc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:25.411536731Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1357,"session_id":"dbb96ce3-2469-4b96-8b5b-c469fce4458e"} {"msg":"Crucible stats registered with UUID: 89564478-016a-4da2-9b5e-d151e4542edc","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:25.411546939Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1357,"session_id":"dbb96ce3-2469-4b96-8b5b-c469fce4458e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:55:25.411556088Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1357,"session_id":"dbb96ce3-2469-4b96-8b5b-c469fce4458e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:25.411693448Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1357,"":"io task","client":"0","":"downstairs","session_id":"dbb96ce3-2469-4b96-8b5b-c469fce4458e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:25.411716814Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1357,"":"io task","client":"2","":"downstairs","session_id":"dbb96ce3-2469-4b96-8b5b-c469fce4458e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:25.411727223Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1357,"":"io task","client":"1","":"downstairs","session_id":"dbb96ce3-2469-4b96-8b5b-c469fce4458e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:25.411887319Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1357,"task":"crutest"} {"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:25.411905786Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1357} {"msg":"89564478-016a-4da2-9b5e-d151e4542edc active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:25.411930323Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1357,"session_id":"dbb96ce3-2469-4b96-8b5b-c469fce4458e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:25.411939611Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1357,"client":"0","":"downstairs","session_id":"dbb96ce3-2469-4b96-8b5b-c469fce4458e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:25.41194927Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1357,"client":"1","":"downstairs","session_id":"dbb96ce3-2469-4b96-8b5b-c469fce4458e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:25.411959208Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1357,"client":"2","":"downstairs","session_id":"dbb96ce3-2469-4b96-8b5b-c469fce4458e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:25.411968837Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1357,"":"io task","client":"2","":"downstairs","session_id":"dbb96ce3-2469-4b96-8b5b-c469fce4458e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:25.411978745Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1357,"":"io task","client":"0","":"downstairs","session_id":"dbb96ce3-2469-4b96-8b5b-c469fce4458e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:25.411988614Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1357,"":"io task","client":"1","":"downstairs","session_id":"dbb96ce3-2469-4b96-8b5b-c469fce4458e"} {"msg":"ds_connection connected from Ok(127.0.0.1:51396)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:25.412042986Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1357,"":"io task","client":"1","":"downstairs","session_id":"dbb96ce3-2469-4b96-8b5b-c469fce4458e"} {"msg":"ds_connection connected from Ok(127.0.0.1:39951)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:25.412054134Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1357,"":"io task","client":"2","":"downstairs","session_id":"dbb96ce3-2469-4b96-8b5b-c469fce4458e"} {"msg":"ds_connection connected from Ok(127.0.0.1:60376)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:25.412108306Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1357,"":"io task","client":"0","":"downstairs","session_id":"dbb96ce3-2469-4b96-8b5b-c469fce4458e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:25.412766979Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1357,"client":"1","":"downstairs","session_id":"dbb96ce3-2469-4b96-8b5b-c469fce4458e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:25.412787656Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1357,"client":"2","":"downstairs","session_id":"dbb96ce3-2469-4b96-8b5b-c469fce4458e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:25.412874703Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1357,"client":"0","":"downstairs","session_id":"dbb96ce3-2469-4b96-8b5b-c469fce4458e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:25.413108819Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1357,"session_id":"dbb96ce3-2469-4b96-8b5b-c469fce4458e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:25.413132875Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1357,"session_id":"dbb96ce3-2469-4b96-8b5b-c469fce4458e"} {"msg":"[0]R flush_numbers[0..12]: [78, 102, 101, 88, 102, 103, 97, 95, 104, 100, 102, 97]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:25.413187267Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1357,"":"downstairs","":"downstairs","session_id":"dbb96ce3-2469-4b96-8b5b-c469fce4458e"} {"msg":"[0]R generation[0..12]: [43, 53, 52, 47, 53, 54, 50, 49, 55, 52, 53, 50]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:25.413199135Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1357,"":"downstairs","":"downstairs","session_id":"dbb96ce3-2469-4b96-8b5b-c469fce4458e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:25.413210394Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1357,"":"downstairs","":"downstairs","session_id":"dbb96ce3-2469-4b96-8b5b-c469fce4458e"} {"msg":"[1]R flush_numbers[0..12]: [78, 102, 101, 88, 102, 103, 97, 95, 104, 100, 102, 97]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:25.413221072Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1357,"":"downstairs","":"downstairs","session_id":"dbb96ce3-2469-4b96-8b5b-c469fce4458e"} {"msg":"[1]R generation[0..12]: [43, 53, 52, 47, 53, 54, 50, 49, 55, 52, 53, 50]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:25.413231661Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1357,"":"downstairs","":"downstairs","session_id":"dbb96ce3-2469-4b96-8b5b-c469fce4458e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:25.413242789Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1357,"":"downstairs","":"downstairs","session_id":"dbb96ce3-2469-4b96-8b5b-c469fce4458e"} {"msg":"[2]R flush_numbers[0..12]: [78, 102, 101, 88, 102, 103, 97, 95, 104, 100, 102, 97]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:25.413253367Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1357,"":"downstairs","":"downstairs","session_id":"dbb96ce3-2469-4b96-8b5b-c469fce4458e"} {"msg":"[2]R generation[0..12]: [43, 53, 52, 47, 53, 54, 50, 49, 55, 52, 53, 50]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:25.413263866Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1357,"":"downstairs","":"downstairs","session_id":"dbb96ce3-2469-4b96-8b5b-c469fce4458e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:25.413274954Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1357,"":"downstairs","":"downstairs","session_id":"dbb96ce3-2469-4b96-8b5b-c469fce4458e"} {"msg":"Max found gen is 56","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:25.413284593Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1357,"":"downstairs","":"downstairs","session_id":"dbb96ce3-2469-4b96-8b5b-c469fce4458e"} {"msg":"Generation requested: 56 >= found:56","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:25.413294321Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1357,"":"downstairs","":"downstairs","session_id":"dbb96ce3-2469-4b96-8b5b-c469fce4458e"} {"msg":"Next flush: 105","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:25.41330674Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1357,"":"downstairs","":"downstairs","session_id":"dbb96ce3-2469-4b96-8b5b-c469fce4458e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:25.413316798Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1357,"":"downstairs","":"downstairs","session_id":"dbb96ce3-2469-4b96-8b5b-c469fce4458e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:25.413326537Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1357,"session_id":"dbb96ce3-2469-4b96-8b5b-c469fce4458e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:25.413335345Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1357,"client":"0","":"downstairs","session_id":"dbb96ce3-2469-4b96-8b5b-c469fce4458e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:25.413345284Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1357,"client":"1","":"downstairs","session_id":"dbb96ce3-2469-4b96-8b5b-c469fce4458e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:25.413354832Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1357,"client":"2","":"downstairs","session_id":"dbb96ce3-2469-4b96-8b5b-c469fce4458e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:25.413364541Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1357,"session_id":"dbb96ce3-2469-4b96-8b5b-c469fce4458e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:25.41337379Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1357,"session_id":"dbb96ce3-2469-4b96-8b5b-c469fce4458e"} {"msg":"89564478-016a-4da2-9b5e-d151e4542edc is now active with session: dbb96ce3-2469-4b96-8b5b-c469fce4458e","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:25.413383748Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1357,"session_id":"dbb96ce3-2469-4b96-8b5b-c469fce4458e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:25.413392587Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1357,"session_id":"dbb96ce3-2469-4b96-8b5b-c469fce4458e"} {"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:25.413401816Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1357} {"msg":"Activated sub_volume 89564478-016a-4da2-9b5e-d151e4542edc","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:25.413410104Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1357} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:25.413418283Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1357,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:25.413733127Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1357,"task":"crutest"} 01/30 Write block 365 len 16384 data: 7 6 4 4 02/30 Read block 356 len 8192 03/30 Write block 376 len 28672 data: 4 4 3 3 2 2 3 04/30 Write block 114 len 40960 data: 8 8 7 7 6 5 4 4 3 3 05/30 Read block 522 len 24576 06/30 Write block 24 len 36864 data: 10 10 9 7 7 7 5 6 5 07/30 Read block 14 len 4096 08/30 Read block 232 len 12288 09/30 Write block 5 len 12288 data: 3 3 3 10/30 Flush 11/30 Write block 457 len 40960 data: 6 6 6 4 4 5 5 6 6 8 12/30 Read block 267 len 16384 13/30 Read block 172 len 16384 14/30 Read block 214 len 4096 15/30 Read block 490 len 12288 16/30 Read block 351 len 36864 17/30 Read block 500 len 16384 18/30 Write block 35 len 12288 data: 6 6 7 19/30 Read block 563 len 16384 20/30 Read block 275 len 4096 21/30 Flush 22/30 Write block 289 len 8192 data: 5 5 23/30 Read block 177 len 4096 24/30 Write block 277 len 4096 data: 6 25/30 Flush 26/30 Write block 472 len 16384 data: 8 6 8 8 27/30 Read block 245 len 40960 28/30 Read block 15 len 32768 29/30 Read block 379 len 40960 30/30 Write block 365 len 24576 data: 8 7 5 5 4 4 ---------------------------------------------------------------- Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Sent false 1001 Acked Read 2 Done Done Sent false 1002 Acked Write 7 Done Done Sent false 1003 Acked Write 10 Done Done Sent false 1004 Acked Read 6 Done Done Sent false 1005 Acked Write 9 Done Done Sent false 1006 Acked Read 1 Done Done Sent false 1007 Acked Read 3 Done Done Sent false 1008 Acked Write 3 Done Done Sent false 1009 Acked Flush 0 Done Done Sent false 1010 Acked Write 10 Done Done Sent false 1011 Acked Read 4 Done Done Sent false 1012 Acked Read 4 Done Done Sent false 1013 Acked Read 1 Done Done Sent false 1014 Acked Read 3 Done Done Sent false 1015 Acked Read 9 Done Done Sent false 1016 Acked Read 4 Done Done Sent false 1017 Acked Write 3 Done Done Sent false 1018 Acked Read 4 Done Done Sent false 1019 Acked Read 1 Done Done Sent false 1020 Acked Flush 0 Done Done Sent false 1021 Acked Write 2 Done Done Sent false 1022 Acked Read 1 Done Done Sent false 1023 Acked Write 1 Done Done Sent false 1024 Acked Flush 0 Done Done Sent false 1025 Acked Write 4 Done Done Sent false 1026 Acked Read 10 Done Done Sent false 1027 Acked Read 8 Done Done Sent false 1028 Acked Read 10 Done Done Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 30 32 Done 29 29 0 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:55:25.422542277Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1357} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:25.422767614Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1357,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 56 56 43 105 105 78 F F F <--- 1 020-039 56 56 53 106 106 102 F F F <--- 2 040-059 52 52 52 101 101 101 F F F 3 060-079 47 47 47 88 88 88 F F F 4 080-099 53 53 53 102 102 102 F F F 5 100-119 56 56 54 105 105 103 F F F <--- 6 120-139 56 56 50 105 105 97 F F F <--- 7 140-159 49 49 49 95 95 95 F F F 8 160-179 55 55 55 104 104 104 F F F 9 180-199 52 52 52 100 100 100 F F F 10 200-219 53 53 53 102 102 102 F F F 11 220-239 50 50 50 97 97 97 F F F 12 240-259 55 55 55 104 104 104 F F F 13 260-279 56 56 53 107 107 102 F F F <--- 14 280-299 56 56 52 107 107 99 F F F <--- 15 300-319 49 49 49 95 95 95 F F F 16 320-339 55 55 55 104 104 104 F F F 17 340-359 55 55 55 104 104 104 F F F 18 360-379 56 56 55 105 105 104 T T F <--- 19 380-399 56 56 54 105 105 103 F F F <--- 20 400-419 55 55 55 104 104 104 F F F 21 420-439 55 55 55 104 104 104 F F F 22 440-459 56 56 52 106 106 101 F F F <--- 23 460-479 56 56 55 106 106 104 T T F <--- 24 480-499 41 41 41 76 76 76 F F F 25 500-519 48 48 48 93 93 93 F F F 26 520-539 54 54 54 103 103 103 F F F 27 540-559 55 55 55 104 104 104 F F F 28 560-579 55 55 55 104 104 104 F F F 29 580-599 52 52 52 100 100 100 F F F Max gen: 56, Max flush: 107 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 28 Verifying data now No change after verify Loop: 28 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 56 56 56 105 105 105 F F F 1 020-039 56 56 56 106 106 106 F F F 2 040-059 52 52 52 101 101 101 F F F 3 060-079 47 47 47 88 88 88 F F F 4 080-099 53 53 53 102 102 102 F F F 5 100-119 56 56 56 105 105 105 F F F 6 120-139 56 56 56 105 105 105 F F F 7 140-159 49 49 49 95 95 95 F F F 8 160-179 55 55 55 104 104 104 F F F 9 180-199 52 52 52 100 100 100 F F F 10 200-219 53 53 53 102 102 102 F F F 11 220-239 50 50 50 97 97 97 F F F 12 240-259 55 55 55 104 104 104 F F F 13 260-279 56 56 56 107 107 107 F F F 14 280-299 56 56 56 107 107 107 F F F 15 300-319 49 49 49 95 95 95 F F F 16 320-339 55 55 55 104 104 104 F F F 17 340-359 55 55 55 104 104 104 F F F 18 360-379 57 57 57 108 108 108 F F F 19 380-399 56 56 56 105 105 105 F F F 20 400-419 55 55 55 104 104 104 F F F 21 420-439 55 55 55 104 104 104 F F F 22 440-459 56 56 56 106 106 106 F F F 23 460-479 57 57 57 108 108 108 F F F 24 480-499 41 41 41 76 76 76 F F F 25 500-519 48 48 48 93 93 93 F F F 26 520-539 54 54 54 103 103 103 F F F 27 540-559 55 55 55 104 104 104 F F F 28 560-579 55 55 55 104 104 104 F F F 29 580-599 52 52 52 100 100 100 F F F Max gen: 57, Max flush: 108 Begin loop 29 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:29.661940575Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1365,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:55:29.669028359Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1365,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:29.669078162Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1365,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:29.675930521Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1365,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:29.676201661Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1365} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:29.676231146Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1365} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:29.676239465Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1365} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:29.676250733Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1365} {"msg":"Crucible 89fe3aa1-600a-47f5-87e3-16773c8aec4a has session id: 9f0a74d7-2a08-4578-a920-e99f83f803ee","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:29.676608911Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1365,"session_id":"9f0a74d7-2a08-4578-a920-e99f83f803ee"} {"msg":"Upstairs opts: Upstairs UUID: 89fe3aa1-600a-47f5-87e3-16773c8aec4a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:29.676638726Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1365,"session_id":"9f0a74d7-2a08-4578-a920-e99f83f803ee"} {"msg":"Crucible stats registered with UUID: 89fe3aa1-600a-47f5-87e3-16773c8aec4a","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:29.676649705Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1365,"session_id":"9f0a74d7-2a08-4578-a920-e99f83f803ee"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:55:29.676658923Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1365,"session_id":"9f0a74d7-2a08-4578-a920-e99f83f803ee"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:29.676783435Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1365,"":"io task","client":"0","":"downstairs","session_id":"9f0a74d7-2a08-4578-a920-e99f83f803ee"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:29.676800432Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1365,"":"io task","client":"1","":"downstairs","session_id":"9f0a74d7-2a08-4578-a920-e99f83f803ee"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:29.676810701Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1365,"":"io task","client":"2","":"downstairs","session_id":"9f0a74d7-2a08-4578-a920-e99f83f803ee"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:29.676994684Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1365,"task":"crutest"} {"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:29.677011581Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1365} {"msg":"89fe3aa1-600a-47f5-87e3-16773c8aec4a active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:29.677030398Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1365,"session_id":"9f0a74d7-2a08-4578-a920-e99f83f803ee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:29.677039477Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1365,"client":"0","":"downstairs","session_id":"9f0a74d7-2a08-4578-a920-e99f83f803ee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:29.677049496Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1365,"client":"1","":"downstairs","session_id":"9f0a74d7-2a08-4578-a920-e99f83f803ee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:29.677059164Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1365,"client":"2","":"downstairs","session_id":"9f0a74d7-2a08-4578-a920-e99f83f803ee"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:29.677068943Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1365,"":"io task","client":"2","":"downstairs","session_id":"9f0a74d7-2a08-4578-a920-e99f83f803ee"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:29.677079271Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1365,"":"io task","client":"0","":"downstairs","session_id":"9f0a74d7-2a08-4578-a920-e99f83f803ee"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:29.67708936Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1365,"":"io task","client":"1","":"downstairs","session_id":"9f0a74d7-2a08-4578-a920-e99f83f803ee"} {"msg":"ds_connection connected from Ok(127.0.0.1:50274)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:29.677206373Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1365,"":"io task","client":"2","":"downstairs","session_id":"9f0a74d7-2a08-4578-a920-e99f83f803ee"} {"msg":"ds_connection connected from Ok(127.0.0.1:44799)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:29.677250506Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1365,"":"io task","client":"1","":"downstairs","session_id":"9f0a74d7-2a08-4578-a920-e99f83f803ee"} {"msg":"ds_connection connected from Ok(127.0.0.1:55246)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:29.677261494Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1365,"":"io task","client":"0","":"downstairs","session_id":"9f0a74d7-2a08-4578-a920-e99f83f803ee"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:29.677806804Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1365,"client":"2","":"downstairs","session_id":"9f0a74d7-2a08-4578-a920-e99f83f803ee"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:29.67783453Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1365,"client":"1","":"downstairs","session_id":"9f0a74d7-2a08-4578-a920-e99f83f803ee"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:29.677935665Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1365,"client":"0","":"downstairs","session_id":"9f0a74d7-2a08-4578-a920-e99f83f803ee"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:29.678112649Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1365,"session_id":"9f0a74d7-2a08-4578-a920-e99f83f803ee"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:29.678128187Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1365,"session_id":"9f0a74d7-2a08-4578-a920-e99f83f803ee"} {"msg":"[0]R flush_numbers[0..12]: [105, 106, 101, 88, 102, 105, 105, 95, 104, 100, 102, 97]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:29.678222283Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1365,"":"downstairs","":"downstairs","session_id":"9f0a74d7-2a08-4578-a920-e99f83f803ee"} {"msg":"[0]R generation[0..12]: [56, 56, 52, 47, 53, 56, 56, 49, 55, 52, 53, 50]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:29.678236351Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1365,"":"downstairs","":"downstairs","session_id":"9f0a74d7-2a08-4578-a920-e99f83f803ee"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:29.678247669Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1365,"":"downstairs","":"downstairs","session_id":"9f0a74d7-2a08-4578-a920-e99f83f803ee"} {"msg":"[1]R flush_numbers[0..12]: [105, 106, 101, 88, 102, 105, 105, 95, 104, 100, 102, 97]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:29.678258257Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1365,"":"downstairs","":"downstairs","session_id":"9f0a74d7-2a08-4578-a920-e99f83f803ee"} {"msg":"[1]R generation[0..12]: [56, 56, 52, 47, 53, 56, 56, 49, 55, 52, 53, 50]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:29.678268756Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1365,"":"downstairs","":"downstairs","session_id":"9f0a74d7-2a08-4578-a920-e99f83f803ee"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:29.678279664Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1365,"":"downstairs","":"downstairs","session_id":"9f0a74d7-2a08-4578-a920-e99f83f803ee"} {"msg":"[2]R flush_numbers[0..12]: [105, 106, 101, 88, 102, 105, 105, 95, 104, 100, 102, 97]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:29.678290363Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1365,"":"downstairs","":"downstairs","session_id":"9f0a74d7-2a08-4578-a920-e99f83f803ee"} {"msg":"[2]R generation[0..12]: [56, 56, 52, 47, 53, 56, 56, 49, 55, 52, 53, 50]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:29.678300741Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1365,"":"downstairs","":"downstairs","session_id":"9f0a74d7-2a08-4578-a920-e99f83f803ee"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:29.678311729Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1365,"":"downstairs","":"downstairs","session_id":"9f0a74d7-2a08-4578-a920-e99f83f803ee"} {"msg":"Max found gen is 58","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:29.678321228Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1365,"":"downstairs","":"downstairs","session_id":"9f0a74d7-2a08-4578-a920-e99f83f803ee"} {"msg":"Generation requested: 58 >= found:58","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:29.678330947Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1365,"":"downstairs","":"downstairs","session_id":"9f0a74d7-2a08-4578-a920-e99f83f803ee"} {"msg":"Next flush: 109","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:29.678340405Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1365,"":"downstairs","":"downstairs","session_id":"9f0a74d7-2a08-4578-a920-e99f83f803ee"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:29.678349764Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1365,"":"downstairs","":"downstairs","session_id":"9f0a74d7-2a08-4578-a920-e99f83f803ee"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:29.678359472Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1365,"session_id":"9f0a74d7-2a08-4578-a920-e99f83f803ee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:29.678368241Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1365,"client":"0","":"downstairs","session_id":"9f0a74d7-2a08-4578-a920-e99f83f803ee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:29.678379359Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1365,"client":"1","":"downstairs","session_id":"9f0a74d7-2a08-4578-a920-e99f83f803ee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:29.678389038Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1365,"client":"2","":"downstairs","session_id":"9f0a74d7-2a08-4578-a920-e99f83f803ee"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:29.678398907Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1365,"session_id":"9f0a74d7-2a08-4578-a920-e99f83f803ee"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:29.678408185Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1365,"session_id":"9f0a74d7-2a08-4578-a920-e99f83f803ee"} {"msg":"89fe3aa1-600a-47f5-87e3-16773c8aec4a is now active with session: 9f0a74d7-2a08-4578-a920-e99f83f803ee","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:29.678418044Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1365,"session_id":"9f0a74d7-2a08-4578-a920-e99f83f803ee"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:29.678426792Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1365,"session_id":"9f0a74d7-2a08-4578-a920-e99f83f803ee"} {"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:29.678435811Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1365} {"msg":"Activated sub_volume 89fe3aa1-600a-47f5-87e3-16773c8aec4a","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:29.678449229Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1365} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:29.678457528Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1365,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:29.678747355Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1365,"task":"crutest"} 01/30 Write block 391 len 12288 data: 7 8 7 02/30 Read block 192 len 24576 03/30 Read block 566 len 40960 04/30 Write block 271 len 32768 data: 9 8 6 7 8 7 7 8 05/30 Read block 580 len 20480 06/30 Write block 300 len 8192 data: 4 5 07/30 Read block 286 len 16384 08/30 Write block 529 len 20480 data: 4 4 5 6 7 09/30 Read block 62 len 24576 10/30 Write block 578 len 20480 data: 5 5 4 8 6 11/30 Flush 12/30 Write block 126 len 8192 data: 3 4 13/30 Read block 63 len 20480 14/30 Read block 495 len 12288 15/30 Read block 343 len 24576 16/30 Read block 95 len 12288 17/30 Write block 140 len 24576 data: 8 8 7 6 7 7 18/30 Write block 427 len 4096 data: 7 19/30 Write block 129 len 8192 data: 4 3 20/30 Read block 406 len 4096 21/30 Write block 435 len 16384 data: 6 5 5 6 22/30 Read block 456 len 16384 23/30 Read block 22 len 24576 24/30 Read block 360 len 28672 25/30 Write block 8 len 4096 data: 3 26/30 Write block 288 len 20480 data: 4 6 6 5 5 27/30 Write block 499 len 16384 data: 4 4 4 6 28/30 Read block 209 len 32768 29/30 Read block 510 len 40960 30/30 Write block 123 len 4096 data: 4 ---------------------------------------------------------------- Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Sent Done Done false 1001 Acked Read 6 Sent Done Done false 1002 Acked Read 10 Sent Done Done false 1003 Acked Write 8 Sent Done Done false 1004 Acked Read 5 Sent Done Done false 1005 Acked Write 2 Sent Done Done false 1006 Acked Read 4 Sent Done Done false 1007 Acked Write 5 Sent Done Done false 1008 Acked Read 6 Sent Done Done false 1009 Acked Write 5 Sent Done Done false 1010 Acked Flush 0 Sent Done Done false 1011 Acked Write 2 Sent Done Done false 1012 Acked Read 5 Sent Done Done false 1013 Acked Read 3 Sent Done Done false 1014 Acked Read 6 Sent Done Done false 1015 Acked Read 3 Sent Done Done false 1016 Acked Write 6 Sent Done Done false 1017 Acked Write 1 Sent Done Done false 1018 Acked Write 2 Sent Done Done false 1019 Acked Read 1 Sent Done Done false 1020 Acked Write 4 Sent Done Done false 1021 Acked Read 4 Sent Done Done false 1022 Acked Read 6 Sent Done Done false 1023 Acked Read 7 Sent Done Done false 1024 Acked Write 1 Sent Done Done false 1025 Acked Write 5 Sent Done Done false 1026 Acked Write 4 Sent Done Done false 1027 Acked Read 8 Sent Done Done false 1028 Acked Read 10 Sent Done Done false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 1 1 32 Done 0 29 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1010)) Some(JobId(1010)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:55:29.687075216Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1365} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:29.687263059Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1365,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 56 56 56 105 105 105 F T T <--- 1 020-039 56 56 56 106 106 106 F F F 2 040-059 52 52 52 101 101 101 F F F 3 060-079 47 47 47 88 88 88 F F F 4 080-099 53 53 53 102 102 102 F F F 5 100-119 56 56 56 105 105 105 F F F 6 120-139 56 56 56 105 105 105 F T T <--- 7 140-159 49 49 49 95 95 95 F T T <--- 8 160-179 55 55 55 104 104 104 F F F 9 180-199 52 52 52 100 100 100 F F F 10 200-219 53 53 53 102 102 102 F F F 11 220-239 50 50 50 97 97 97 F F F 12 240-259 55 55 55 104 104 104 F F F 13 260-279 56 58 58 107 109 109 F F F <--- 14 280-299 56 56 56 107 107 107 F T T <--- 15 300-319 49 58 58 95 109 109 F F F <--- 16 320-339 55 55 55 104 104 104 F F F 17 340-359 55 55 55 104 104 104 F F F 18 360-379 57 57 57 108 108 108 F F F 19 380-399 56 58 58 105 109 109 F F F <--- 20 400-419 55 55 55 104 104 104 F F F 21 420-439 55 55 55 104 104 104 F T T <--- 22 440-459 56 56 56 106 106 106 F F F 23 460-479 57 57 57 108 108 108 F F F 24 480-499 41 41 41 76 76 76 F T T <--- 25 500-519 48 48 48 93 93 93 F T T <--- 26 520-539 54 58 58 103 109 109 F F F <--- 27 540-559 55 55 55 104 104 104 F F F 28 560-579 55 58 58 104 109 109 F F F <--- 29 580-599 52 58 58 100 109 109 F F F <--- Max gen: 58, Max flush: 109 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 29 Verifying data now No change after verify Loop: 29 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 59 59 59 110 110 110 F F F 1 020-039 56 56 56 106 106 106 F F F 2 040-059 52 52 52 101 101 101 F F F 3 060-079 47 47 47 88 88 88 F F F 4 080-099 53 53 53 102 102 102 F F F 5 100-119 56 56 56 105 105 105 F F F 6 120-139 59 59 59 110 110 110 F F F 7 140-159 59 59 59 110 110 110 F F F 8 160-179 55 55 55 104 104 104 F F F 9 180-199 52 52 52 100 100 100 F F F 10 200-219 53 53 53 102 102 102 F F F 11 220-239 50 50 50 97 97 97 F F F 12 240-259 55 55 55 104 104 104 F F F 13 260-279 58 58 58 109 109 109 F F F 14 280-299 59 59 59 110 110 110 F F F 15 300-319 58 58 58 109 109 109 F F F 16 320-339 55 55 55 104 104 104 F F F 17 340-359 55 55 55 104 104 104 F F F 18 360-379 57 57 57 108 108 108 F F F 19 380-399 58 58 58 109 109 109 F F F 20 400-419 55 55 55 104 104 104 F F F 21 420-439 59 59 59 110 110 110 F F F 22 440-459 56 56 56 106 106 106 F F F 23 460-479 57 57 57 108 108 108 F F F 24 480-499 59 59 59 110 110 110 F F F 25 500-519 59 59 59 110 110 110 F F F 26 520-539 58 58 58 109 109 109 F F F 27 540-559 55 55 55 104 104 104 F F F 28 560-579 58 58 58 109 109 109 F F F 29 580-599 58 58 58 109 109 109 F F F Max gen: 59, Max flush: 110 Begin loop 30 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:33.948630715Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1373,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:55:33.955758893Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1373,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:33.955807916Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1373,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:33.962590595Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1373,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:33.962849967Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1373} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:33.962877363Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1373} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:33.962889701Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1373} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:33.962901359Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1373} {"msg":"Crucible 3ff99486-b497-42cc-9027-8da3384f8ab8 has session id: 776a7112-6143-47ff-9b5f-050113d4b198","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:33.963201855Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1373,"session_id":"776a7112-6143-47ff-9b5f-050113d4b198"} {"msg":"Upstairs opts: Upstairs UUID: 3ff99486-b497-42cc-9027-8da3384f8ab8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:33.963221782Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1373,"session_id":"776a7112-6143-47ff-9b5f-050113d4b198"} {"msg":"Crucible stats registered with UUID: 3ff99486-b497-42cc-9027-8da3384f8ab8","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:33.96323173Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1373,"session_id":"776a7112-6143-47ff-9b5f-050113d4b198"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:55:33.963240969Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1373,"session_id":"776a7112-6143-47ff-9b5f-050113d4b198"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:33.963400196Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1373,"":"io task","client":"2","":"downstairs","session_id":"776a7112-6143-47ff-9b5f-050113d4b198"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:33.963431531Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1373,"":"io task","client":"1","":"downstairs","session_id":"776a7112-6143-47ff-9b5f-050113d4b198"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:33.963442699Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1373,"":"io task","client":"0","":"downstairs","session_id":"776a7112-6143-47ff-9b5f-050113d4b198"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:33.963541035Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1373,"task":"crutest"} {"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:33.963555333Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1373} {"msg":"3ff99486-b497-42cc-9027-8da3384f8ab8 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:33.963588888Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1373,"session_id":"776a7112-6143-47ff-9b5f-050113d4b198"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:33.963598266Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1373,"client":"0","":"downstairs","session_id":"776a7112-6143-47ff-9b5f-050113d4b198"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:33.963608245Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1373,"client":"1","":"downstairs","session_id":"776a7112-6143-47ff-9b5f-050113d4b198"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:33.963618113Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1373,"client":"2","":"downstairs","session_id":"776a7112-6143-47ff-9b5f-050113d4b198"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:33.963630192Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1373,"":"io task","client":"2","":"downstairs","session_id":"776a7112-6143-47ff-9b5f-050113d4b198"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:33.96364046Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1373,"":"io task","client":"0","":"downstairs","session_id":"776a7112-6143-47ff-9b5f-050113d4b198"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:33.963650869Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1373,"":"io task","client":"1","":"downstairs","session_id":"776a7112-6143-47ff-9b5f-050113d4b198"} {"msg":"ds_connection connected from Ok(127.0.0.1:41378)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:33.963744665Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1373,"":"io task","client":"0","":"downstairs","session_id":"776a7112-6143-47ff-9b5f-050113d4b198"} {"msg":"ds_connection connected from Ok(127.0.0.1:58376)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:33.963769951Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1373,"":"io task","client":"1","":"downstairs","session_id":"776a7112-6143-47ff-9b5f-050113d4b198"} {"msg":"ds_connection connected from Ok(127.0.0.1:35673)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:33.963781249Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1373,"":"io task","client":"2","":"downstairs","session_id":"776a7112-6143-47ff-9b5f-050113d4b198"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:33.964477737Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1373,"client":"1","":"downstairs","session_id":"776a7112-6143-47ff-9b5f-050113d4b198"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:33.964516321Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1373,"client":"2","":"downstairs","session_id":"776a7112-6143-47ff-9b5f-050113d4b198"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:33.964530889Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1373,"client":"0","":"downstairs","session_id":"776a7112-6143-47ff-9b5f-050113d4b198"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:33.96479639Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1373,"session_id":"776a7112-6143-47ff-9b5f-050113d4b198"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:33.964816157Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1373,"session_id":"776a7112-6143-47ff-9b5f-050113d4b198"} {"msg":"[0]R flush_numbers[0..12]: [110, 106, 101, 88, 102, 105, 110, 110, 104, 100, 102, 97]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:33.964830295Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1373,"":"downstairs","":"downstairs","session_id":"776a7112-6143-47ff-9b5f-050113d4b198"} {"msg":"[0]R generation[0..12]: [59, 56, 52, 47, 53, 56, 59, 59, 55, 52, 53, 50]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:33.964841083Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1373,"":"downstairs","":"downstairs","session_id":"776a7112-6143-47ff-9b5f-050113d4b198"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:33.964852451Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1373,"":"downstairs","":"downstairs","session_id":"776a7112-6143-47ff-9b5f-050113d4b198"} {"msg":"[1]R flush_numbers[0..12]: [110, 106, 101, 88, 102, 105, 110, 110, 104, 100, 102, 97]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:33.964863Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1373,"":"downstairs","":"downstairs","session_id":"776a7112-6143-47ff-9b5f-050113d4b198"} {"msg":"[1]R generation[0..12]: [59, 56, 52, 47, 53, 56, 59, 59, 55, 52, 53, 50]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:33.964876968Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1373,"":"downstairs","":"downstairs","session_id":"776a7112-6143-47ff-9b5f-050113d4b198"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:33.964888166Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1373,"":"downstairs","":"downstairs","session_id":"776a7112-6143-47ff-9b5f-050113d4b198"} {"msg":"[2]R flush_numbers[0..12]: [110, 106, 101, 88, 102, 105, 110, 110, 104, 100, 102, 97]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:33.964898754Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1373,"":"downstairs","":"downstairs","session_id":"776a7112-6143-47ff-9b5f-050113d4b198"} {"msg":"[2]R generation[0..12]: [59, 56, 52, 47, 53, 56, 59, 59, 55, 52, 53, 50]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:33.964909113Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1373,"":"downstairs","":"downstairs","session_id":"776a7112-6143-47ff-9b5f-050113d4b198"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:33.964920151Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1373,"":"downstairs","":"downstairs","session_id":"776a7112-6143-47ff-9b5f-050113d4b198"} {"msg":"Max found gen is 60","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:33.96492972Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1373,"":"downstairs","":"downstairs","session_id":"776a7112-6143-47ff-9b5f-050113d4b198"} {"msg":"Generation requested: 60 >= found:60","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:33.964939568Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1373,"":"downstairs","":"downstairs","session_id":"776a7112-6143-47ff-9b5f-050113d4b198"} {"msg":"Next flush: 111","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:33.964948827Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1373,"":"downstairs","":"downstairs","session_id":"776a7112-6143-47ff-9b5f-050113d4b198"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:33.964958196Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1373,"":"downstairs","":"downstairs","session_id":"776a7112-6143-47ff-9b5f-050113d4b198"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:33.964968004Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1373,"session_id":"776a7112-6143-47ff-9b5f-050113d4b198"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:33.964976683Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1373,"client":"0","":"downstairs","session_id":"776a7112-6143-47ff-9b5f-050113d4b198"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:33.964986322Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1373,"client":"1","":"downstairs","session_id":"776a7112-6143-47ff-9b5f-050113d4b198"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:33.96499611Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1373,"client":"2","":"downstairs","session_id":"776a7112-6143-47ff-9b5f-050113d4b198"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:33.965016757Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1373,"session_id":"776a7112-6143-47ff-9b5f-050113d4b198"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:33.965025786Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1373,"session_id":"776a7112-6143-47ff-9b5f-050113d4b198"} {"msg":"3ff99486-b497-42cc-9027-8da3384f8ab8 is now active with session: 776a7112-6143-47ff-9b5f-050113d4b198","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:33.965037094Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1373,"session_id":"776a7112-6143-47ff-9b5f-050113d4b198"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:33.965045973Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1373,"session_id":"776a7112-6143-47ff-9b5f-050113d4b198"} {"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:33.965055131Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1373} {"msg":"Activated sub_volume 3ff99486-b497-42cc-9027-8da3384f8ab8","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:33.9650631Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1373} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:33.965071339Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1373,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:33.965361756Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1373,"task":"crutest"} 01/30 Write block 187 len 24576 data: 4 3 4 5 10 11 02/30 Write block 283 len 4096 data: 5 03/30 Write block 186 len 40960 data: 3 5 4 5 6 11 12 10 9 7 04/30 Read block 416 len 32768 05/30 Read block 177 len 20480 06/30 Read block 572 len 24576 07/30 Read block 588 len 40960 08/30 Read block 77 len 8192 09/30 Read block 143 len 32768 10/30 Write block 205 len 12288 data: 6 6 9 11/30 Read block 124 len 40960 12/30 Write block 326 len 36864 data: 8 7 6 6 7 7 6 5 5 13/30 Write block 125 len 8192 data: 3 4 14/30 Write block 488 len 36864 data: 6 4 5 4 4 3 4 4 4 15/30 Read block 34 len 8192 16/30 Write block 571 len 32768 data: 3 3 4 5 5 5 5 6 17/30 Read block 474 len 32768 18/30 Read block 81 len 36864 19/30 Read block 296 len 20480 20/30 Write block 343 len 16384 data: 3 3 3 4 21/30 Read block 559 len 24576 22/30 Write block 270 len 8192 data: 8 10 23/30 Read block 0 len 16384 24/30 Read block 310 len 4096 25/30 Write block 469 len 4096 data: 7 26/30 Write block 381 len 36864 data: 3 4 3 3 4 5 5 5 4 27/30 Read block 421 len 4096 28/30 Write block 220 len 16384 data: 5 5 5 4 29/30 Write block 379 len 4096 data: 4 30/30 Write block 134 len 4096 data: 8 ---------------------------------------------------------------- Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Write 1 Sent Done Done false 1002 Acked Write 10 Sent Done Done false 1003 Acked Read 8 Sent Done Done false 1004 Acked Read 5 Sent Done Done false 1005 Acked Read 6 Sent Done Done false 1006 Acked Read 10 Sent Done Done false 1007 Acked Read 2 Sent Done Done false 1008 Acked Read 8 Sent Done Done false 1009 Acked Write 3 Sent Done Done false 1010 Acked Read 10 Sent Done Done false 1011 Acked Write 9 Sent Done Done false 1012 Acked Write 2 Sent Done Done false 1013 Acked Write 9 Sent Done Done false 1014 Acked Read 2 Sent Done Done false 1015 Acked Write 8 Sent Done Done false 1016 Acked Read 8 Sent Done Done false 1017 Acked Read 9 Sent Done Done false 1018 Acked Read 5 Sent Done Done false 1019 Acked Write 4 Sent Done Done false 1020 Acked Read 6 Sent Done Done false 1021 Acked Write 2 Sent Done Done false 1022 Acked Read 4 Sent Done Done false 1023 Acked Read 1 Sent Done Done false 1024 Acked Write 1 Sent Done Done false 1025 Acked Write 9 Sent Done Done false 1026 Acked Read 1 Sent Done Done false 1027 Acked Write 4 Sent Sent Sent false 1028 Acked Write 1 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 3 3 35 Done 1 27 27 55 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:55:33.973180953Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1373} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:33.973394111Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1373,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 59 59 59 110 110 110 F F F 1 020-039 56 56 56 106 106 106 F F F 2 040-059 52 52 52 101 101 101 F F F 3 060-079 47 47 47 88 88 88 F F F 4 080-099 53 53 53 102 102 102 F F F 5 100-119 56 56 56 105 105 105 F F F 6 120-139 59 59 59 110 110 110 F T T <--- 7 140-159 59 59 59 110 110 110 F F F 8 160-179 55 55 55 104 104 104 F F F 9 180-199 52 52 52 100 100 100 T T T 10 200-219 53 53 53 102 102 102 F T T <--- 11 220-239 50 50 50 97 97 97 F T T <--- 12 240-259 55 55 55 104 104 104 F F F 13 260-279 58 58 58 109 109 109 F T T <--- 14 280-299 59 59 59 110 110 110 F T T <--- 15 300-319 58 58 58 109 109 109 F F F 16 320-339 55 55 55 104 104 104 F T T <--- 17 340-359 55 55 55 104 104 104 F T T <--- 18 360-379 57 57 57 108 108 108 F T T <--- 19 380-399 58 58 58 109 109 109 F T T <--- 20 400-419 55 55 55 104 104 104 F F F 21 420-439 59 59 59 110 110 110 F F F 22 440-459 56 56 56 106 106 106 F F F 23 460-479 57 57 57 108 108 108 F T T <--- 24 480-499 59 59 59 110 110 110 F T T <--- 25 500-519 59 59 59 110 110 110 F F F 26 520-539 58 58 58 109 109 109 F F F 27 540-559 55 55 55 104 104 104 F F F 28 560-579 58 58 58 109 109 109 F T T <--- 29 580-599 58 58 58 109 109 109 F F F Max gen: 59, Max flush: 110 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 30 Verifying data now Files /tmp/test_repair/test_repair_verify.data and /tmp/test_repair/test_repair_verify.data.last differ 189,190d188 < 2, < 4, 191a190 > 5, 194c193 < 10, --- > 6, 195a195,196 > 12, > 10, 197,198c198 < 8, < 6, --- > 7, diff found after verify Loop: 30 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 59 59 59 110 110 110 F F F 1 020-039 56 56 56 106 106 106 F F F 2 040-059 52 52 52 101 101 101 F F F 3 060-079 47 47 47 88 88 88 F F F 4 080-099 53 53 53 102 102 102 F F F 5 100-119 56 56 56 105 105 105 F F F 6 120-139 60 60 60 111 111 111 F F F 7 140-159 59 59 59 110 110 110 F F F 8 160-179 55 55 55 104 104 104 F F F 9 180-199 60 60 60 111 111 111 F F F 10 200-219 60 60 60 111 111 111 F F F 11 220-239 60 60 60 111 111 111 F F F 12 240-259 55 55 55 104 104 104 F F F 13 260-279 60 60 60 111 111 111 F F F 14 280-299 60 60 60 111 111 111 F F F 15 300-319 58 58 58 109 109 109 F F F 16 320-339 60 60 60 111 111 111 F F F 17 340-359 60 60 60 111 111 111 F F F 18 360-379 60 60 60 111 111 111 F F F 19 380-399 60 60 60 111 111 111 F F F 20 400-419 55 55 55 104 104 104 F F F 21 420-439 59 59 59 110 110 110 F F F 22 440-459 56 56 56 106 106 106 F F F 23 460-479 60 60 60 111 111 111 F F F 24 480-499 60 60 60 111 111 111 F F F 25 500-519 59 59 59 110 110 110 F F F 26 520-539 58 58 58 109 109 109 F F F 27 540-559 55 55 55 104 104 104 F F F 28 560-579 60 60 60 111 111 111 F F F 29 580-599 58 58 58 109 109 109 F F F Max gen: 60, Max flush: 111 Begin loop 31 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:38.239729025Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1384,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:38.247767669Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1384,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:38.248061805Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1384} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:38.248087052Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1384} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:38.24809591Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1384} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:38.248107829Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1384} {"msg":"Crucible def19dd3-17ad-44d7-bdf0-082b088f5f71 has session id: 5c859bc6-9543-47cc-8baf-7ffddd499ba0","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:38.24843642Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1384,"session_id":"5c859bc6-9543-47cc-8baf-7ffddd499ba0"} {"msg":"Upstairs opts: Upstairs UUID: def19dd3-17ad-44d7-bdf0-082b088f5f71, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:38.248465126Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1384,"session_id":"5c859bc6-9543-47cc-8baf-7ffddd499ba0"} {"msg":"Crucible stats registered with UUID: def19dd3-17ad-44d7-bdf0-082b088f5f71","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:38.248478934Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1384,"session_id":"5c859bc6-9543-47cc-8baf-7ffddd499ba0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:55:38.248488303Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1384,"session_id":"5c859bc6-9543-47cc-8baf-7ffddd499ba0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:38.24863772Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1384,"":"io task","client":"2","":"downstairs","session_id":"5c859bc6-9543-47cc-8baf-7ffddd499ba0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:38.248668336Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1384,"":"io task","client":"0","":"downstairs","session_id":"5c859bc6-9543-47cc-8baf-7ffddd499ba0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:38.248679644Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1384,"":"io task","client":"1","":"downstairs","session_id":"5c859bc6-9543-47cc-8baf-7ffddd499ba0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:38.24877927Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1384,"task":"crutest"} {"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:38.248789608Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1384} {"msg":"def19dd3-17ad-44d7-bdf0-082b088f5f71 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:38.248823903Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1384,"session_id":"5c859bc6-9543-47cc-8baf-7ffddd499ba0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:38.248833552Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1384,"client":"0","":"downstairs","session_id":"5c859bc6-9543-47cc-8baf-7ffddd499ba0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:38.24884343Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1384,"client":"1","":"downstairs","session_id":"5c859bc6-9543-47cc-8baf-7ffddd499ba0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:38.248853019Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1384,"client":"2","":"downstairs","session_id":"5c859bc6-9543-47cc-8baf-7ffddd499ba0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:38.248862677Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1384,"":"io task","client":"2","":"downstairs","session_id":"5c859bc6-9543-47cc-8baf-7ffddd499ba0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:38.248872906Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1384,"":"io task","client":"0","":"downstairs","session_id":"5c859bc6-9543-47cc-8baf-7ffddd499ba0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:38.248883224Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1384,"":"io task","client":"1","":"downstairs","session_id":"5c859bc6-9543-47cc-8baf-7ffddd499ba0"} {"msg":"ds_connection connected from Ok(127.0.0.1:56795)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:38.248938326Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1384,"":"io task","client":"2","":"downstairs","session_id":"5c859bc6-9543-47cc-8baf-7ffddd499ba0"} {"msg":"ds_connection connected from Ok(127.0.0.1:55869)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:38.248964012Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1384,"":"io task","client":"1","":"downstairs","session_id":"5c859bc6-9543-47cc-8baf-7ffddd499ba0"} {"msg":"ds_connection connected from Ok(127.0.0.1:38485)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:38.249040951Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1384,"":"io task","client":"0","":"downstairs","session_id":"5c859bc6-9543-47cc-8baf-7ffddd499ba0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:38.249585951Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1384,"client":"0","":"downstairs","session_id":"5c859bc6-9543-47cc-8baf-7ffddd499ba0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:38.249598039Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1384,"client":"1","":"downstairs","session_id":"5c859bc6-9543-47cc-8baf-7ffddd499ba0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:38.249619046Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1384,"client":"2","":"downstairs","session_id":"5c859bc6-9543-47cc-8baf-7ffddd499ba0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:38.249892715Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1384,"session_id":"5c859bc6-9543-47cc-8baf-7ffddd499ba0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:38.249902364Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1384,"session_id":"5c859bc6-9543-47cc-8baf-7ffddd499ba0"} {"msg":"[0]R flush_numbers[0..12]: [110, 106, 101, 88, 102, 105, 111, 110, 104, 111, 111, 111]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:38.249936879Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1384,"":"downstairs","":"downstairs","session_id":"5c859bc6-9543-47cc-8baf-7ffddd499ba0"} {"msg":"[0]R generation[0..12]: [59, 56, 52, 47, 53, 56, 60, 59, 55, 60, 60, 60]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:38.249948097Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1384,"":"downstairs","":"downstairs","session_id":"5c859bc6-9543-47cc-8baf-7ffddd499ba0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:38.249959246Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1384,"":"downstairs","":"downstairs","session_id":"5c859bc6-9543-47cc-8baf-7ffddd499ba0"} {"msg":"[1]R flush_numbers[0..12]: [110, 106, 101, 88, 102, 105, 111, 110, 104, 111, 111, 111]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:38.249969904Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1384,"":"downstairs","":"downstairs","session_id":"5c859bc6-9543-47cc-8baf-7ffddd499ba0"} {"msg":"[1]R generation[0..12]: [59, 56, 52, 47, 53, 56, 60, 59, 55, 60, 60, 60]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:38.249980462Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1384,"":"downstairs","":"downstairs","session_id":"5c859bc6-9543-47cc-8baf-7ffddd499ba0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:38.249991401Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1384,"":"downstairs","":"downstairs","session_id":"5c859bc6-9543-47cc-8baf-7ffddd499ba0"} {"msg":"[2]R flush_numbers[0..12]: [110, 106, 101, 88, 102, 105, 111, 110, 104, 111, 111, 111]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:38.250002059Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1384,"":"downstairs","":"downstairs","session_id":"5c859bc6-9543-47cc-8baf-7ffddd499ba0"} {"msg":"[2]R generation[0..12]: [59, 56, 52, 47, 53, 56, 60, 59, 55, 60, 60, 60]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:38.250012778Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1384,"":"downstairs","":"downstairs","session_id":"5c859bc6-9543-47cc-8baf-7ffddd499ba0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:38.250025996Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1384,"":"downstairs","":"downstairs","session_id":"5c859bc6-9543-47cc-8baf-7ffddd499ba0"} {"msg":"Max found gen is 61","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:38.250035514Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1384,"":"downstairs","":"downstairs","session_id":"5c859bc6-9543-47cc-8baf-7ffddd499ba0"} {"msg":"Generation requested: 62 >= found:61","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:38.250045043Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1384,"":"downstairs","":"downstairs","session_id":"5c859bc6-9543-47cc-8baf-7ffddd499ba0"} {"msg":"Next flush: 112","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:38.250054541Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1384,"":"downstairs","":"downstairs","session_id":"5c859bc6-9543-47cc-8baf-7ffddd499ba0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:38.25006411Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1384,"":"downstairs","":"downstairs","session_id":"5c859bc6-9543-47cc-8baf-7ffddd499ba0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:38.250074069Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1384,"session_id":"5c859bc6-9543-47cc-8baf-7ffddd499ba0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:38.250082947Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1384,"client":"0","":"downstairs","session_id":"5c859bc6-9543-47cc-8baf-7ffddd499ba0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:38.250092756Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1384,"client":"1","":"downstairs","session_id":"5c859bc6-9543-47cc-8baf-7ffddd499ba0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:38.250102414Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1384,"client":"2","":"downstairs","session_id":"5c859bc6-9543-47cc-8baf-7ffddd499ba0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:38.250112133Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1384,"session_id":"5c859bc6-9543-47cc-8baf-7ffddd499ba0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:38.250121272Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1384,"session_id":"5c859bc6-9543-47cc-8baf-7ffddd499ba0"} {"msg":"def19dd3-17ad-44d7-bdf0-082b088f5f71 is now active with session: 5c859bc6-9543-47cc-8baf-7ffddd499ba0","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:38.25013124Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1384,"session_id":"5c859bc6-9543-47cc-8baf-7ffddd499ba0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:38.250140139Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1384,"session_id":"5c859bc6-9543-47cc-8baf-7ffddd499ba0"} {"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:38.250149307Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1384} {"msg":"Activated sub_volume def19dd3-17ad-44d7-bdf0-082b088f5f71","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:38.250157626Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1384} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:38.250165765Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1384,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:38.250458382Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1384,"task":"crutest"} 01/30 Write block 286 len 4096 data: 7 02/30 Write block 302 len 32768 data: 3 3 3 5 5 3 3 5 03/30 Write block 354 len 16384 data: 7 7 8 8 04/30 Read block 124 len 12288 05/30 Write block 375 len 16384 data: 5 5 5 4 06/30 Read block 30 len 20480 07/30 Read block 270 len 24576 08/30 Read block 76 len 24576 09/30 Write block 83 len 28672 data: 2 4 5 5 5 7 6 10/30 Write block 593 len 8192 data: 9 8 11/30 Read block 456 len 24576 12/30 Write block 363 len 20480 data: 6 7 9 8 6 13/30 Write block 365 len 40960 data: 10 9 7 6 5 5 5 6 5 5 14/30 Write block 434 len 40960 data: 6 7 6 6 7 5 3 4 4 4 15/30 Write block 99 len 28672 data: 4 5 6 4 4 4 5 16/30 Write block 339 len 12288 data: 6 5 4 17/30 Flush 18/30 Write block 25 len 24576 data: 11 10 8 8 8 6 19/30 Read block 540 len 20480 20/30 Read block 535 len 16384 21/30 Write block 290 len 20480 data: 7 6 6 5 4 22/30 Read block 551 len 20480 23/30 Read block 88 len 12288 24/30 Write block 116 len 8192 data: 8 8 25/30 Read block 181 len 36864 26/30 Write block 194 len 24576 data: 9 7 6 7 7 6 27/30 Write block 5 len 24576 data: 4 4 4 4 3 3 28/30 Read block 492 len 40960 29/30 Read block 500 len 36864 30/30 Write block 100 len 16384 data: 6 7 5 5 ---------------------------------------------------------------- Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Write 8 Done Done Done false 1002 Acked Write 4 Done Done Done false 1003 Acked Read 3 Done Done Done false 1004 Acked Write 4 Done Done Done false 1005 Acked Read 5 Done Sent Done false 1006 Acked Read 6 Done Sent Done false 1007 Acked Read 6 Done Sent Done false 1008 Acked Write 7 Done Sent Done false 1009 Acked Write 2 Done Sent Done false 1010 Acked Read 6 Done Sent Done false 1011 Acked Write 5 Done Sent Done false 1012 Acked Write 10 Done Sent Done false 1013 Acked Write 10 Done Sent Done false 1014 Acked Write 7 Done Sent Done false 1015 Acked Write 3 Done Sent Done false 1016 Acked Flush 0 Done Sent Done false 1017 Acked Write 6 Done Sent Done false 1018 Acked Read 5 Done Sent Done false 1019 Acked Read 4 Done Sent Done false 1020 Acked Write 5 Done Sent Done false 1021 Acked Read 5 Done Sent Done false 1022 Acked Read 3 Done Sent Done false 1023 Acked Write 2 Done Sent Done false 1024 Acked Read 9 Done Sent Done false 1025 Acked Write 6 Done Sent Done false 1026 Acked Write 6 Done Sent Done false 1027 Acked Read 10 Done Sent Done false 1028 Acked Read 9 Done Sent Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 25 1 27 Done 29 5 29 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) None Some(JobId(1016)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:55:38.258716673Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1384} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:38.258887978Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1384,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 59 59 59 110 110 110 T F T <--- 1 020-039 56 56 56 106 106 106 T F T <--- 2 040-059 52 52 52 101 101 101 F F F 3 060-079 47 47 47 88 88 88 F F F 4 080-099 62 53 62 112 102 112 F F F <--- 5 100-119 62 56 62 112 105 112 T F T <--- 6 120-139 60 60 60 111 111 111 F F F 7 140-159 59 59 59 110 110 110 F F F 8 160-179 55 55 55 104 104 104 F F F 9 180-199 60 60 60 111 111 111 T F T <--- 10 200-219 60 60 60 111 111 111 F F F 11 220-239 60 60 60 111 111 111 F F F 12 240-259 55 55 55 104 104 104 F F F 13 260-279 60 60 60 111 111 111 F F F 14 280-299 62 60 62 112 111 112 T T T <--- 15 300-319 62 58 62 112 109 112 F T F <--- 16 320-339 62 60 62 112 111 112 F F F <--- 17 340-359 62 60 62 112 111 112 F T F <--- 18 360-379 62 60 62 112 111 112 F T F <--- 19 380-399 60 60 60 111 111 111 F F F 20 400-419 55 55 55 104 104 104 F F F 21 420-439 62 59 62 112 110 112 F F F <--- 22 440-459 62 56 62 112 106 112 F F F <--- 23 460-479 60 60 60 111 111 111 F F F 24 480-499 60 60 60 111 111 111 F F F 25 500-519 59 59 59 110 110 110 F F F 26 520-539 58 58 58 109 109 109 F F F 27 540-559 55 55 55 104 104 104 F F F 28 560-579 60 60 60 111 111 111 F F F 29 580-599 62 58 62 112 109 112 F F F <--- Max gen: 62, Max flush: 112 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 31 Verifying data now No change after verify Loop: 31 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 63 63 63 113 113 113 F F F 1 020-039 63 63 63 113 113 113 F F F 2 040-059 52 52 52 101 101 101 F F F 3 060-079 47 47 47 88 88 88 F F F 4 080-099 62 62 62 112 112 112 F F F 5 100-119 63 63 63 113 113 113 F F F 6 120-139 60 60 60 111 111 111 F F F 7 140-159 59 59 59 110 110 110 F F F 8 160-179 55 55 55 104 104 104 F F F 9 180-199 63 63 63 113 113 113 F F F 10 200-219 60 60 60 111 111 111 F F F 11 220-239 60 60 60 111 111 111 F F F 12 240-259 55 55 55 104 104 104 F F F 13 260-279 60 60 60 111 111 111 F F F 14 280-299 63 63 63 113 113 113 F F F 15 300-319 62 62 62 112 112 112 F F F 16 320-339 62 62 62 112 112 112 F F F 17 340-359 62 62 62 112 112 112 F F F 18 360-379 62 62 62 112 112 112 F F F 19 380-399 60 60 60 111 111 111 F F F 20 400-419 55 55 55 104 104 104 F F F 21 420-439 62 62 62 112 112 112 F F F 22 440-459 62 62 62 112 112 112 F F F 23 460-479 60 60 60 111 111 111 F F F 24 480-499 60 60 60 111 111 111 F F F 25 500-519 59 59 59 110 110 110 F F F 26 520-539 58 58 58 109 109 109 F F F 27 540-559 55 55 55 104 104 104 F F F 28 560-579 60 60 60 111 111 111 F F F 29 580-599 62 62 62 112 112 112 F F F Max gen: 63, Max flush: 113 Begin loop 32 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:42.50781809Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1392,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:42.515853174Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1392,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:42.516142052Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1392} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:42.516170967Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1392} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:42.516179486Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1392} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:42.516190884Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1392} {"msg":"Crucible 25c82a26-3613-435e-8432-b800d7a80faf has session id: 0e354b64-d875-419d-ab01-bfb01ae8d8cc","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:42.516514926Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1392,"session_id":"0e354b64-d875-419d-ab01-bfb01ae8d8cc"} {"msg":"Upstairs opts: Upstairs UUID: 25c82a26-3613-435e-8432-b800d7a80faf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:42.516545752Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1392,"session_id":"0e354b64-d875-419d-ab01-bfb01ae8d8cc"} {"msg":"Crucible stats registered with UUID: 25c82a26-3613-435e-8432-b800d7a80faf","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:42.516576257Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1392,"session_id":"0e354b64-d875-419d-ab01-bfb01ae8d8cc"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:55:42.516617741Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1392,"session_id":"0e354b64-d875-419d-ab01-bfb01ae8d8cc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:42.516712087Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1392,"":"io task","client":"2","":"downstairs","session_id":"0e354b64-d875-419d-ab01-bfb01ae8d8cc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:42.516729145Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1392,"":"io task","client":"0","":"downstairs","session_id":"0e354b64-d875-419d-ab01-bfb01ae8d8cc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:42.516742173Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1392,"":"io task","client":"1","":"downstairs","session_id":"0e354b64-d875-419d-ab01-bfb01ae8d8cc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:42.516868704Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1392,"task":"crutest"} {"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:42.516887542Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1392} {"msg":"25c82a26-3613-435e-8432-b800d7a80faf active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:42.516917227Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1392,"session_id":"0e354b64-d875-419d-ab01-bfb01ae8d8cc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:42.516926686Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1392,"client":"0","":"downstairs","session_id":"0e354b64-d875-419d-ab01-bfb01ae8d8cc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:42.516936574Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1392,"client":"1","":"downstairs","session_id":"0e354b64-d875-419d-ab01-bfb01ae8d8cc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:42.516946333Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1392,"client":"2","":"downstairs","session_id":"0e354b64-d875-419d-ab01-bfb01ae8d8cc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:42.516955971Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1392,"":"io task","client":"2","":"downstairs","session_id":"0e354b64-d875-419d-ab01-bfb01ae8d8cc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:42.51696612Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1392,"":"io task","client":"0","":"downstairs","session_id":"0e354b64-d875-419d-ab01-bfb01ae8d8cc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:42.516976448Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1392,"":"io task","client":"1","":"downstairs","session_id":"0e354b64-d875-419d-ab01-bfb01ae8d8cc"} {"msg":"ds_connection connected from Ok(127.0.0.1:61368)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:42.517060736Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1392,"":"io task","client":"0","":"downstairs","session_id":"0e354b64-d875-419d-ab01-bfb01ae8d8cc"} {"msg":"ds_connection connected from Ok(127.0.0.1:50207)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:42.517086472Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1392,"":"io task","client":"2","":"downstairs","session_id":"0e354b64-d875-419d-ab01-bfb01ae8d8cc"} {"msg":"ds_connection connected from Ok(127.0.0.1:37220)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:42.517144404Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1392,"":"io task","client":"1","":"downstairs","session_id":"0e354b64-d875-419d-ab01-bfb01ae8d8cc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:42.517808796Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1392,"client":"2","":"downstairs","session_id":"0e354b64-d875-419d-ab01-bfb01ae8d8cc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:42.517836422Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1392,"client":"1","":"downstairs","session_id":"0e354b64-d875-419d-ab01-bfb01ae8d8cc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:42.5178474Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1392,"client":"0","":"downstairs","session_id":"0e354b64-d875-419d-ab01-bfb01ae8d8cc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:42.518113971Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1392,"session_id":"0e354b64-d875-419d-ab01-bfb01ae8d8cc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:42.518151585Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1392,"session_id":"0e354b64-d875-419d-ab01-bfb01ae8d8cc"} {"msg":"[0]R flush_numbers[0..12]: [113, 113, 101, 88, 112, 113, 111, 110, 104, 113, 111, 111]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:42.518200918Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1392,"":"downstairs","":"downstairs","session_id":"0e354b64-d875-419d-ab01-bfb01ae8d8cc"} {"msg":"[0]R generation[0..12]: [63, 63, 52, 47, 62, 63, 60, 59, 55, 63, 60, 60]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:42.518212086Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1392,"":"downstairs","":"downstairs","session_id":"0e354b64-d875-419d-ab01-bfb01ae8d8cc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:42.518223054Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1392,"":"downstairs","":"downstairs","session_id":"0e354b64-d875-419d-ab01-bfb01ae8d8cc"} {"msg":"[1]R flush_numbers[0..12]: [113, 113, 101, 88, 112, 113, 111, 110, 104, 113, 111, 111]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:42.518233823Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1392,"":"downstairs","":"downstairs","session_id":"0e354b64-d875-419d-ab01-bfb01ae8d8cc"} {"msg":"[1]R generation[0..12]: [63, 63, 52, 47, 62, 63, 60, 59, 55, 63, 60, 60]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:42.518244041Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1392,"":"downstairs","":"downstairs","session_id":"0e354b64-d875-419d-ab01-bfb01ae8d8cc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:42.51825501Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1392,"":"downstairs","":"downstairs","session_id":"0e354b64-d875-419d-ab01-bfb01ae8d8cc"} {"msg":"[2]R flush_numbers[0..12]: [113, 113, 101, 88, 112, 113, 111, 110, 104, 113, 111, 111]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:42.518265768Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1392,"":"downstairs","":"downstairs","session_id":"0e354b64-d875-419d-ab01-bfb01ae8d8cc"} {"msg":"[2]R generation[0..12]: [63, 63, 52, 47, 62, 63, 60, 59, 55, 63, 60, 60]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:42.518276147Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1392,"":"downstairs","":"downstairs","session_id":"0e354b64-d875-419d-ab01-bfb01ae8d8cc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:42.518287075Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1392,"":"downstairs","":"downstairs","session_id":"0e354b64-d875-419d-ab01-bfb01ae8d8cc"} {"msg":"Max found gen is 64","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:42.518296384Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1392,"":"downstairs","":"downstairs","session_id":"0e354b64-d875-419d-ab01-bfb01ae8d8cc"} {"msg":"Generation requested: 64 >= found:64","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:42.518306142Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1392,"":"downstairs","":"downstairs","session_id":"0e354b64-d875-419d-ab01-bfb01ae8d8cc"} {"msg":"Next flush: 114","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:42.518315551Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1392,"":"downstairs","":"downstairs","session_id":"0e354b64-d875-419d-ab01-bfb01ae8d8cc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:42.518326529Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1392,"":"downstairs","":"downstairs","session_id":"0e354b64-d875-419d-ab01-bfb01ae8d8cc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:42.518336548Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1392,"session_id":"0e354b64-d875-419d-ab01-bfb01ae8d8cc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:42.518345326Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1392,"client":"0","":"downstairs","session_id":"0e354b64-d875-419d-ab01-bfb01ae8d8cc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:42.518355385Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1392,"client":"1","":"downstairs","session_id":"0e354b64-d875-419d-ab01-bfb01ae8d8cc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:42.518365303Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1392,"client":"2","":"downstairs","session_id":"0e354b64-d875-419d-ab01-bfb01ae8d8cc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:42.518375242Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1392,"session_id":"0e354b64-d875-419d-ab01-bfb01ae8d8cc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:42.518384491Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1392,"session_id":"0e354b64-d875-419d-ab01-bfb01ae8d8cc"} {"msg":"25c82a26-3613-435e-8432-b800d7a80faf is now active with session: 0e354b64-d875-419d-ab01-bfb01ae8d8cc","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:42.518394349Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1392,"session_id":"0e354b64-d875-419d-ab01-bfb01ae8d8cc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:42.518403118Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1392,"session_id":"0e354b64-d875-419d-ab01-bfb01ae8d8cc"} {"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:42.518412027Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1392} {"msg":"Activated sub_volume 25c82a26-3613-435e-8432-b800d7a80faf","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:42.518420035Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1392} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:42.518428024Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1392,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:42.518746277Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1392,"task":"crutest"} 01/30 Write block 404 len 8192 data: 5 7 02/30 Read block 366 len 20480 03/30 Read block 260 len 12288 04/30 Write block 281 len 8192 data: 4 5 05/30 Write block 302 len 12288 data: 4 4 4 06/30 Read block 158 len 20480 07/30 Write block 475 len 36864 data: 9 7 6 7 7 7 6 6 5 08/30 Write block 38 len 16384 data: 11 11 10 10 09/30 Write block 399 len 12288 data: 3 3 3 10/30 Write block 360 len 16384 data: 8 6 3 7 11/30 Read block 403 len 40960 12/30 Read block 456 len 36864 13/30 Read block 11 len 24576 14/30 Write block 133 len 20480 data: 5 9 8 7 6 15/30 Write block 250 len 28672 data: 5 4 4 4 4 5 5 16/30 Read block 200 len 20480 17/30 Read block 477 len 16384 18/30 Read block 194 len 12288 19/30 Flush 20/30 Write block 177 len 28672 data: 6 6 4 3 4 3 3 21/30 Write block 147 len 32768 data: 6 6 6 6 6 6 7 6 22/30 Flush 23/30 Write block 587 len 4096 data: 7 24/30 Write block 189 len 28672 data: 5 6 11 12 10 10 8 25/30 Write block 495 len 24576 data: 5 5 4 3 5 5 26/30 Write block 207 len 40960 data: 10 8 6 4 5 6 6 6 5 4 27/30 Write block 101 len 4096 data: 8 28/30 Read block 48 len 12288 29/30 Write block 467 len 24576 data: 7 8 8 7 9 9 30/30 Read block 491 len 20480 ---------------------------------------------------------------- Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Read 5 Done Done Done false 1002 Acked Read 3 Done Done Done false 1003 Acked Write 2 Done Done Done false 1004 Acked Write 3 Done Done Done false 1005 Acked Read 5 Done Sent Done false 1006 Acked Write 9 Done Sent Done false 1007 Acked Write 4 Done Sent Done false 1008 Acked Write 3 Done Sent Done false 1009 Acked Write 4 Done Sent Done false 1010 Acked Read 10 Done Sent Done false 1011 Acked Read 9 Done Sent Done false 1012 Acked Read 6 Done Sent Done false 1013 Acked Write 5 Done Sent Done false 1014 Acked Write 7 Done Sent Done false 1015 Acked Read 5 Done Sent Done false 1016 Acked Read 4 Done Sent Done false 1017 Acked Read 3 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Write 7 Done Sent Done false 1020 Acked Write 8 Done Sent Done false 1021 Acked Flush 0 Done Sent Done false 1022 Acked Write 1 Done Sent Done false 1023 Acked Write 7 Done Sent Done false 1024 Acked Write 6 Done Sent Done false 1025 Acked Write 10 Done Sent Done false 1026 Acked Write 1 Done Sent Done false 1027 Acked Read 3 Done Sent Done false 1028 Acked Write 6 Done Sent Done false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 25 1 26 Done 30 5 29 64 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) None Some(JobId(1021)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:55:42.526786841Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1392} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:42.526963045Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1392,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 63 63 63 113 113 113 F F F 1 020-039 64 63 64 114 113 114 F F F <--- 2 040-059 64 52 64 114 101 114 F F F <--- 3 060-079 47 47 47 88 88 88 F F F 4 080-099 62 62 62 112 112 112 F F F 5 100-119 63 63 63 113 113 113 T F T <--- 6 120-139 64 60 64 114 111 114 F F F <--- 7 140-159 64 59 64 115 110 115 F F F <--- 8 160-179 64 55 64 115 104 115 F F F <--- 9 180-199 64 63 64 115 113 115 T F T <--- 10 200-219 60 60 60 111 111 111 T F T <--- 11 220-239 60 60 60 111 111 111 F F F 12 240-259 64 55 64 114 104 114 F F F <--- 13 260-279 60 60 60 111 111 111 F F F 14 280-299 64 63 64 114 113 114 F T F <--- 15 300-319 64 62 64 114 112 114 F T F <--- 16 320-339 62 62 62 112 112 112 F F F 17 340-359 62 62 62 112 112 112 F F F 18 360-379 64 62 64 114 112 114 F F F <--- 19 380-399 64 60 64 114 111 114 F F F <--- 20 400-419 64 55 64 114 104 114 F T F <--- 21 420-439 62 62 62 112 112 112 F F F 22 440-459 62 62 62 112 112 112 F F F 23 460-479 64 60 64 114 111 114 T F T <--- 24 480-499 64 60 64 114 111 114 T F T <--- 25 500-519 59 59 59 110 110 110 T F T <--- 26 520-539 58 58 58 109 109 109 F F F 27 540-559 55 55 55 104 104 104 F F F 28 560-579 60 60 60 111 111 111 F F F 29 580-599 62 62 62 112 112 112 T F T <--- Max gen: 64, Max flush: 115 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 32 Verifying data now No change after verify Loop: 32 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 63 63 63 113 113 113 F F F 1 020-039 64 64 64 114 114 114 F F F 2 040-059 64 64 64 114 114 114 F F F 3 060-079 47 47 47 88 88 88 F F F 4 080-099 62 62 62 112 112 112 F F F 5 100-119 65 65 65 116 116 116 F F F 6 120-139 64 64 64 114 114 114 F F F 7 140-159 64 64 64 115 115 115 F F F 8 160-179 64 64 64 115 115 115 F F F 9 180-199 65 65 65 116 116 116 F F F 10 200-219 65 65 65 116 116 116 F F F 11 220-239 60 60 60 111 111 111 F F F 12 240-259 64 64 64 114 114 114 F F F 13 260-279 60 60 60 111 111 111 F F F 14 280-299 64 64 64 114 114 114 F F F 15 300-319 64 64 64 114 114 114 F F F 16 320-339 62 62 62 112 112 112 F F F 17 340-359 62 62 62 112 112 112 F F F 18 360-379 64 64 64 114 114 114 F F F 19 380-399 64 64 64 114 114 114 F F F 20 400-419 64 64 64 114 114 114 F F F 21 420-439 62 62 62 112 112 112 F F F 22 440-459 62 62 62 112 112 112 F F F 23 460-479 65 65 65 116 116 116 F F F 24 480-499 65 65 65 116 116 116 F F F 25 500-519 65 65 65 116 116 116 F F F 26 520-539 58 58 58 109 109 109 F F F 27 540-559 55 55 55 104 104 104 F F F 28 560-579 60 60 60 111 111 111 F F F 29 580-599 65 65 65 116 116 116 F F F Max gen: 65, Max flush: 116 Begin loop 33 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:46.790992559Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1400,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:46.798999168Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1400,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:46.799290665Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1400} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:46.799316511Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1400} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:46.799327829Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1400} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:46.799339547Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1400} {"msg":"Crucible 26842e3e-1a5e-4f0b-868a-80668647417f has session id: 99163fdf-4850-4c98-90c5-9ab0ccfb002d","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:46.79972466Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1400,"session_id":"99163fdf-4850-4c98-90c5-9ab0ccfb002d"} {"msg":"Upstairs opts: Upstairs UUID: 26842e3e-1a5e-4f0b-868a-80668647417f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:46.799760345Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1400,"session_id":"99163fdf-4850-4c98-90c5-9ab0ccfb002d"} {"msg":"Crucible stats registered with UUID: 26842e3e-1a5e-4f0b-868a-80668647417f","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:46.799771664Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1400,"session_id":"99163fdf-4850-4c98-90c5-9ab0ccfb002d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:55:46.799780682Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1400,"session_id":"99163fdf-4850-4c98-90c5-9ab0ccfb002d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:46.799919242Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1400,"":"io task","client":"0","":"downstairs","session_id":"99163fdf-4850-4c98-90c5-9ab0ccfb002d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:46.799950447Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1400,"":"io task","client":"2","":"downstairs","session_id":"99163fdf-4850-4c98-90c5-9ab0ccfb002d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:46.799961406Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1400,"":"io task","client":"1","":"downstairs","session_id":"99163fdf-4850-4c98-90c5-9ab0ccfb002d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:46.800119012Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1400,"task":"crutest"} {"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:46.80013481Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1400} {"msg":"26842e3e-1a5e-4f0b-868a-80668647417f active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:46.800168155Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1400,"session_id":"99163fdf-4850-4c98-90c5-9ab0ccfb002d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:46.800177824Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1400,"client":"0","":"downstairs","session_id":"99163fdf-4850-4c98-90c5-9ab0ccfb002d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:46.800187812Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1400,"client":"1","":"downstairs","session_id":"99163fdf-4850-4c98-90c5-9ab0ccfb002d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:46.8001997Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1400,"client":"2","":"downstairs","session_id":"99163fdf-4850-4c98-90c5-9ab0ccfb002d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:46.800209479Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1400,"":"io task","client":"2","":"downstairs","session_id":"99163fdf-4850-4c98-90c5-9ab0ccfb002d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:46.800219577Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1400,"":"io task","client":"0","":"downstairs","session_id":"99163fdf-4850-4c98-90c5-9ab0ccfb002d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:46.800229656Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1400,"":"io task","client":"1","":"downstairs","session_id":"99163fdf-4850-4c98-90c5-9ab0ccfb002d"} {"msg":"ds_connection connected from Ok(127.0.0.1:63148)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:46.800283568Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1400,"":"io task","client":"2","":"downstairs","session_id":"99163fdf-4850-4c98-90c5-9ab0ccfb002d"} {"msg":"ds_connection connected from Ok(127.0.0.1:41889)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:46.800308694Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1400,"":"io task","client":"0","":"downstairs","session_id":"99163fdf-4850-4c98-90c5-9ab0ccfb002d"} {"msg":"ds_connection connected from Ok(127.0.0.1:61833)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:46.800395881Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1400,"":"io task","client":"1","":"downstairs","session_id":"99163fdf-4850-4c98-90c5-9ab0ccfb002d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:46.800916745Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1400,"client":"1","":"downstairs","session_id":"99163fdf-4850-4c98-90c5-9ab0ccfb002d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:46.800935882Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1400,"client":"2","":"downstairs","session_id":"99163fdf-4850-4c98-90c5-9ab0ccfb002d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:46.80094709Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1400,"client":"0","":"downstairs","session_id":"99163fdf-4850-4c98-90c5-9ab0ccfb002d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:46.801240727Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1400,"session_id":"99163fdf-4850-4c98-90c5-9ab0ccfb002d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:46.801250395Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1400,"session_id":"99163fdf-4850-4c98-90c5-9ab0ccfb002d"} {"msg":"[0]R flush_numbers[0..12]: [113, 114, 114, 88, 112, 116, 114, 115, 115, 116, 116, 111]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:46.801325424Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1400,"":"downstairs","":"downstairs","session_id":"99163fdf-4850-4c98-90c5-9ab0ccfb002d"} {"msg":"[0]R generation[0..12]: [63, 64, 64, 47, 62, 65, 64, 64, 64, 65, 65, 60]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:46.801336403Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1400,"":"downstairs","":"downstairs","session_id":"99163fdf-4850-4c98-90c5-9ab0ccfb002d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:46.801347661Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1400,"":"downstairs","":"downstairs","session_id":"99163fdf-4850-4c98-90c5-9ab0ccfb002d"} {"msg":"[1]R flush_numbers[0..12]: [113, 114, 114, 88, 112, 116, 114, 115, 115, 116, 116, 111]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:46.801361259Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1400,"":"downstairs","":"downstairs","session_id":"99163fdf-4850-4c98-90c5-9ab0ccfb002d"} {"msg":"[1]R generation[0..12]: [63, 64, 64, 47, 62, 65, 64, 64, 64, 65, 65, 60]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:46.801371847Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1400,"":"downstairs","":"downstairs","session_id":"99163fdf-4850-4c98-90c5-9ab0ccfb002d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:46.801382646Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1400,"":"downstairs","":"downstairs","session_id":"99163fdf-4850-4c98-90c5-9ab0ccfb002d"} {"msg":"[2]R flush_numbers[0..12]: [113, 114, 114, 88, 112, 116, 114, 115, 115, 116, 116, 111]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:46.801393364Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1400,"":"downstairs","":"downstairs","session_id":"99163fdf-4850-4c98-90c5-9ab0ccfb002d"} {"msg":"[2]R generation[0..12]: [63, 64, 64, 47, 62, 65, 64, 64, 64, 65, 65, 60]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:46.801403673Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1400,"":"downstairs","":"downstairs","session_id":"99163fdf-4850-4c98-90c5-9ab0ccfb002d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:46.801414671Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1400,"":"downstairs","":"downstairs","session_id":"99163fdf-4850-4c98-90c5-9ab0ccfb002d"} {"msg":"Max found gen is 66","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:46.80142394Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1400,"":"downstairs","":"downstairs","session_id":"99163fdf-4850-4c98-90c5-9ab0ccfb002d"} {"msg":"Generation requested: 66 >= found:66","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:46.801433748Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1400,"":"downstairs","":"downstairs","session_id":"99163fdf-4850-4c98-90c5-9ab0ccfb002d"} {"msg":"Next flush: 117","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:46.801443247Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1400,"":"downstairs","":"downstairs","session_id":"99163fdf-4850-4c98-90c5-9ab0ccfb002d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:46.801452796Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1400,"":"downstairs","":"downstairs","session_id":"99163fdf-4850-4c98-90c5-9ab0ccfb002d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:46.801462654Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1400,"session_id":"99163fdf-4850-4c98-90c5-9ab0ccfb002d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:46.801471463Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1400,"client":"0","":"downstairs","session_id":"99163fdf-4850-4c98-90c5-9ab0ccfb002d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:46.801481211Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1400,"client":"1","":"downstairs","session_id":"99163fdf-4850-4c98-90c5-9ab0ccfb002d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:46.80149081Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1400,"client":"2","":"downstairs","session_id":"99163fdf-4850-4c98-90c5-9ab0ccfb002d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:46.801500848Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1400,"session_id":"99163fdf-4850-4c98-90c5-9ab0ccfb002d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:46.801511407Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1400,"session_id":"99163fdf-4850-4c98-90c5-9ab0ccfb002d"} {"msg":"26842e3e-1a5e-4f0b-868a-80668647417f is now active with session: 99163fdf-4850-4c98-90c5-9ab0ccfb002d","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:46.801521475Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1400,"session_id":"99163fdf-4850-4c98-90c5-9ab0ccfb002d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:46.801530384Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1400,"session_id":"99163fdf-4850-4c98-90c5-9ab0ccfb002d"} {"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:46.801539513Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1400} {"msg":"Activated sub_volume 26842e3e-1a5e-4f0b-868a-80668647417f","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:46.801547821Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1400} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:46.80155611Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1400,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:46.801834559Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1400,"task":"crutest"} 01/30 Write block 42 len 40960 data: 9 9 8 7 7 7 6 5 5 4 02/30 Write block 344 len 4096 data: 4 03/30 Write block 253 len 24576 data: 5 5 6 6 6 7 04/30 Read block 264 len 36864 05/30 Write block 232 len 16384 data: 3 4 3 3 06/30 Write block 411 len 36864 data: 14 12 13 9 10 11 6 5 4 07/30 Write block 580 len 32768 data: 5 9 7 7 8 8 8 8 08/30 Read block 480 len 20480 09/30 Flush 10/30 Write block 364 len 4096 data: 8 11/30 Write block 146 len 36864 data: 6 7 7 7 7 7 7 8 7 12/30 Read block 371 len 4096 13/30 Read block 329 len 32768 14/30 Read block 416 len 28672 15/30 Read block 71 len 20480 16/30 Read block 517 len 12288 17/30 Read block 8 len 4096 18/30 Write block 35 len 32768 data: 7 7 8 12 12 11 11 10 19/30 Read block 425 len 4096 20/30 Write block 337 len 12288 data: 6 6 7 21/30 Read block 518 len 40960 22/30 Read block 339 len 20480 23/30 Write block 442 len 12288 data: 5 5 7 24/30 Read block 568 len 20480 25/30 Flush 26/30 Write block 370 len 4096 data: 6 27/30 Read block 467 len 12288 28/30 Read block 302 len 32768 29/30 Read block 578 len 8192 30/30 Write block 248 len 20480 data: 7 5 6 5 5 ---------------------------------------------------------------- Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Write 1 Done Done Done false 1002 Acked Write 6 Done Done Sent false 1003 Acked Read 9 Done Done Sent false 1004 Acked Write 4 Done Done Sent false 1005 Acked Write 9 Done Done Sent false 1006 Acked Write 8 Done Done Sent false 1007 Acked Read 5 Done Done Sent false 1008 Acked Flush 0 Done Done Sent false 1009 Acked Write 1 Done Done Sent false 1010 Acked Write 9 Done Done Sent false 1011 Acked Read 1 Done Done Sent false 1012 Acked Read 8 Done Done Sent false 1013 Acked Read 7 Done Done Sent false 1014 Acked Read 5 Done Done Sent false 1015 Acked Read 3 Done Done Sent false 1016 Acked Read 1 Done Done Sent false 1017 Acked Write 8 Done Done Sent false 1018 Acked Read 1 Done Done Sent false 1019 Acked Write 3 Done Done Sent false 1020 Acked Read 10 Done Done Sent false 1021 Acked Read 5 Done Done Sent false 1022 Acked Write 3 Done Done Sent false 1023 Acked Read 5 Done Done Sent false 1024 Acked Flush 0 Done Done Sent false 1025 Acked Write 1 Done Done Sent false 1026 Acked Read 3 Done Done Sent false 1027 Acked Read 8 Done Done Sent false 1028 Acked Read 2 Done Done Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 28 30 Done 29 29 2 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:55:46.810759492Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1400} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:46.810939286Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1400,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 63 63 63 113 113 113 F F F 1 020-039 66 66 64 118 118 114 F F F <--- 2 040-059 66 66 64 118 118 114 F F T <--- 3 060-079 47 47 47 88 88 88 F F F 4 080-099 62 62 62 112 112 112 F F F 5 100-119 65 65 65 116 116 116 F F F 6 120-139 64 64 64 114 114 114 F F F 7 140-159 66 66 64 118 118 115 F F F <--- 8 160-179 64 64 64 115 115 115 F F F 9 180-199 65 65 65 116 116 116 F F F 10 200-219 65 65 65 116 116 116 F F F 11 220-239 66 66 60 117 117 111 F F F <--- 12 240-259 66 66 64 117 117 114 T T F <--- 13 260-279 60 60 60 111 111 111 F F F 14 280-299 64 64 64 114 114 114 F F F 15 300-319 64 64 64 114 114 114 F F F 16 320-339 66 66 62 118 118 112 F F F <--- 17 340-359 66 66 62 117 117 112 F F T <--- 18 360-379 66 66 64 118 118 114 T T F <--- 19 380-399 64 64 64 114 114 114 F F F 20 400-419 66 66 64 117 117 114 F F F <--- 21 420-439 62 62 62 112 112 112 F F F 22 440-459 66 66 62 118 118 112 F F F <--- 23 460-479 65 65 65 116 116 116 F F F 24 480-499 65 65 65 116 116 116 F F F 25 500-519 65 65 65 116 116 116 F F F 26 520-539 58 58 58 109 109 109 F F F 27 540-559 55 55 55 104 104 104 F F F 28 560-579 60 60 60 111 111 111 F F F 29 580-599 66 66 65 117 117 116 F F F <--- Max gen: 66, Max flush: 118 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 33 Verifying data now No change after verify Loop: 33 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 63 63 63 113 113 113 F F F 1 020-039 66 66 66 118 118 118 F F F 2 040-059 66 66 66 118 118 118 F F F 3 060-079 47 47 47 88 88 88 F F F 4 080-099 62 62 62 112 112 112 F F F 5 100-119 65 65 65 116 116 116 F F F 6 120-139 64 64 64 114 114 114 F F F 7 140-159 66 66 66 118 118 118 F F F 8 160-179 64 64 64 115 115 115 F F F 9 180-199 65 65 65 116 116 116 F F F 10 200-219 65 65 65 116 116 116 F F F 11 220-239 66 66 66 117 117 117 F F F 12 240-259 67 67 67 119 119 119 F F F 13 260-279 60 60 60 111 111 111 F F F 14 280-299 64 64 64 114 114 114 F F F 15 300-319 64 64 64 114 114 114 F F F 16 320-339 66 66 66 118 118 118 F F F 17 340-359 66 66 66 117 117 117 F F F 18 360-379 67 67 67 119 119 119 F F F 19 380-399 64 64 64 114 114 114 F F F 20 400-419 66 66 66 117 117 117 F F F 21 420-439 62 62 62 112 112 112 F F F 22 440-459 66 66 66 118 118 118 F F F 23 460-479 65 65 65 116 116 116 F F F 24 480-499 65 65 65 116 116 116 F F F 25 500-519 65 65 65 116 116 116 F F F 26 520-539 58 58 58 109 109 109 F F F 27 540-559 55 55 55 104 104 104 F F F 28 560-579 60 60 60 111 111 111 F F F 29 580-599 66 66 66 117 117 117 F F F Max gen: 67, Max flush: 119 Begin loop 34 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:51.055274465Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1408,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:51.063408435Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1408,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:51.063704741Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1408} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:51.063730517Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1408} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:51.063738876Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1408} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:51.063750314Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1408} {"msg":"Crucible e1528705-6303-42ec-a01b-b5d94ca1dd34 has session id: ebc9fbfd-3dd6-4a6b-b639-5e0c696ea57f","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:51.064106222Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1408,"session_id":"ebc9fbfd-3dd6-4a6b-b639-5e0c696ea57f"} {"msg":"Upstairs opts: Upstairs UUID: e1528705-6303-42ec-a01b-b5d94ca1dd34, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:51.064162743Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1408,"session_id":"ebc9fbfd-3dd6-4a6b-b639-5e0c696ea57f"} {"msg":"Crucible stats registered with UUID: e1528705-6303-42ec-a01b-b5d94ca1dd34","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:51.064176671Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1408,"session_id":"ebc9fbfd-3dd6-4a6b-b639-5e0c696ea57f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:55:51.06418579Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1408,"session_id":"ebc9fbfd-3dd6-4a6b-b639-5e0c696ea57f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:51.064335118Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1408,"":"io task","client":"2","":"downstairs","session_id":"ebc9fbfd-3dd6-4a6b-b639-5e0c696ea57f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:51.064368613Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1408,"":"io task","client":"0","":"downstairs","session_id":"ebc9fbfd-3dd6-4a6b-b639-5e0c696ea57f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:51.064379671Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1408,"":"io task","client":"1","":"downstairs","session_id":"ebc9fbfd-3dd6-4a6b-b639-5e0c696ea57f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:51.064483096Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1408,"task":"crutest"} {"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:51.064498464Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1408} {"msg":"e1528705-6303-42ec-a01b-b5d94ca1dd34 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:51.064537278Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1408,"session_id":"ebc9fbfd-3dd6-4a6b-b639-5e0c696ea57f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:51.064546777Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1408,"client":"0","":"downstairs","session_id":"ebc9fbfd-3dd6-4a6b-b639-5e0c696ea57f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:51.064556595Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1408,"client":"1","":"downstairs","session_id":"ebc9fbfd-3dd6-4a6b-b639-5e0c696ea57f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:51.064566234Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1408,"client":"2","":"downstairs","session_id":"ebc9fbfd-3dd6-4a6b-b639-5e0c696ea57f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:51.064575902Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1408,"":"io task","client":"2","":"downstairs","session_id":"ebc9fbfd-3dd6-4a6b-b639-5e0c696ea57f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:51.064585761Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1408,"":"io task","client":"0","":"downstairs","session_id":"ebc9fbfd-3dd6-4a6b-b639-5e0c696ea57f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:51.064595709Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1408,"":"io task","client":"1","":"downstairs","session_id":"ebc9fbfd-3dd6-4a6b-b639-5e0c696ea57f"} {"msg":"ds_connection connected from Ok(127.0.0.1:42465)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:51.064668869Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1408,"":"io task","client":"0","":"downstairs","session_id":"ebc9fbfd-3dd6-4a6b-b639-5e0c696ea57f"} {"msg":"ds_connection connected from Ok(127.0.0.1:61532)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:51.064695395Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1408,"":"io task","client":"2","":"downstairs","session_id":"ebc9fbfd-3dd6-4a6b-b639-5e0c696ea57f"} {"msg":"ds_connection connected from Ok(127.0.0.1:34780)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:51.064753186Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1408,"":"io task","client":"1","":"downstairs","session_id":"ebc9fbfd-3dd6-4a6b-b639-5e0c696ea57f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:51.065393102Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1408,"client":"0","":"downstairs","session_id":"ebc9fbfd-3dd6-4a6b-b639-5e0c696ea57f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:51.06540613Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1408,"client":"1","":"downstairs","session_id":"ebc9fbfd-3dd6-4a6b-b639-5e0c696ea57f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:51.065417288Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1408,"client":"2","":"downstairs","session_id":"ebc9fbfd-3dd6-4a6b-b639-5e0c696ea57f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:51.065719844Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1408,"session_id":"ebc9fbfd-3dd6-4a6b-b639-5e0c696ea57f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:51.065729532Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1408,"session_id":"ebc9fbfd-3dd6-4a6b-b639-5e0c696ea57f"} {"msg":"[0]R flush_numbers[0..12]: [113, 118, 118, 88, 112, 116, 114, 118, 115, 116, 116, 117]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:51.065765807Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1408,"":"downstairs","":"downstairs","session_id":"ebc9fbfd-3dd6-4a6b-b639-5e0c696ea57f"} {"msg":"[0]R generation[0..12]: [63, 66, 66, 47, 62, 65, 64, 66, 64, 65, 65, 66]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:51.065776885Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1408,"":"downstairs","":"downstairs","session_id":"ebc9fbfd-3dd6-4a6b-b639-5e0c696ea57f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:51.065787994Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1408,"":"downstairs","":"downstairs","session_id":"ebc9fbfd-3dd6-4a6b-b639-5e0c696ea57f"} {"msg":"[1]R flush_numbers[0..12]: [113, 118, 118, 88, 112, 116, 114, 118, 115, 116, 116, 117]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:51.065798652Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1408,"":"downstairs","":"downstairs","session_id":"ebc9fbfd-3dd6-4a6b-b639-5e0c696ea57f"} {"msg":"[1]R generation[0..12]: [63, 66, 66, 47, 62, 65, 64, 66, 64, 65, 65, 66]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:51.06580898Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1408,"":"downstairs","":"downstairs","session_id":"ebc9fbfd-3dd6-4a6b-b639-5e0c696ea57f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:51.065819999Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1408,"":"downstairs","":"downstairs","session_id":"ebc9fbfd-3dd6-4a6b-b639-5e0c696ea57f"} {"msg":"[2]R flush_numbers[0..12]: [113, 118, 118, 88, 112, 116, 114, 118, 115, 116, 116, 117]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:51.065830637Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1408,"":"downstairs","":"downstairs","session_id":"ebc9fbfd-3dd6-4a6b-b639-5e0c696ea57f"} {"msg":"[2]R generation[0..12]: [63, 66, 66, 47, 62, 65, 64, 66, 64, 65, 65, 66]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:51.065841066Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1408,"":"downstairs","":"downstairs","session_id":"ebc9fbfd-3dd6-4a6b-b639-5e0c696ea57f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:51.065853764Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1408,"":"downstairs","":"downstairs","session_id":"ebc9fbfd-3dd6-4a6b-b639-5e0c696ea57f"} {"msg":"Max found gen is 68","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:51.065863242Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1408,"":"downstairs","":"downstairs","session_id":"ebc9fbfd-3dd6-4a6b-b639-5e0c696ea57f"} {"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:51.065872851Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1408,"":"downstairs","":"downstairs","session_id":"ebc9fbfd-3dd6-4a6b-b639-5e0c696ea57f"} {"msg":"Next flush: 120","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:51.06588217Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1408,"":"downstairs","":"downstairs","session_id":"ebc9fbfd-3dd6-4a6b-b639-5e0c696ea57f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:51.065891728Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1408,"":"downstairs","":"downstairs","session_id":"ebc9fbfd-3dd6-4a6b-b639-5e0c696ea57f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:51.065901177Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1408,"session_id":"ebc9fbfd-3dd6-4a6b-b639-5e0c696ea57f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:51.065909896Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1408,"client":"0","":"downstairs","session_id":"ebc9fbfd-3dd6-4a6b-b639-5e0c696ea57f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:51.065919334Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1408,"client":"1","":"downstairs","session_id":"ebc9fbfd-3dd6-4a6b-b639-5e0c696ea57f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:51.065928973Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1408,"client":"2","":"downstairs","session_id":"ebc9fbfd-3dd6-4a6b-b639-5e0c696ea57f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:51.065938651Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1408,"session_id":"ebc9fbfd-3dd6-4a6b-b639-5e0c696ea57f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:51.06594765Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1408,"session_id":"ebc9fbfd-3dd6-4a6b-b639-5e0c696ea57f"} {"msg":"e1528705-6303-42ec-a01b-b5d94ca1dd34 is now active with session: ebc9fbfd-3dd6-4a6b-b639-5e0c696ea57f","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:51.065957489Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1408,"session_id":"ebc9fbfd-3dd6-4a6b-b639-5e0c696ea57f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:51.065966167Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1408,"session_id":"ebc9fbfd-3dd6-4a6b-b639-5e0c696ea57f"} {"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:51.065975286Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1408} {"msg":"Activated sub_volume e1528705-6303-42ec-a01b-b5d94ca1dd34","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:51.065983285Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1408} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:51.065991274Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1408,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:51.066312666Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1408,"task":"crutest"} 01/30 Write block 437 len 20480 data: 7 8 6 4 5 02/30 Write block 86 len 32768 data: 6 6 8 7 7 6 6 6 03/30 Read block 549 len 8192 04/30 Write block 513 len 32768 data: 7 7 8 6 7 9 9 7 05/30 Write block 99 len 12288 data: 5 7 9 06/30 Write block 321 len 28672 data: 6 7 8 8 9 9 8 07/30 Read block 23 len 12288 08/30 Read block 454 len 40960 09/30 Read block 120 len 24576 10/30 Write block 179 len 28672 data: 5 4 5 4 4 5 3 11/30 Write block 595 len 12288 data: 7 5 3 12/30 Write block 44 len 20480 data: 9 8 8 8 7 13/30 Read block 129 len 4096 14/30 Read block 98 len 20480 15/30 Write block 54 len 4096 data: 4 16/30 Write block 156 len 40960 data: 5 6 5 5 5 5 5 5 4 5 17/30 Read block 427 len 36864 18/30 Read block 367 len 28672 19/30 Flush 20/30 Write block 82 len 4096 data: 2 21/30 Write block 449 len 12288 data: 7 6 7 22/30 Read block 35 len 24576 23/30 Write block 230 len 4096 data: 3 24/30 Write block 31 len 12288 data: 7 6 4 25/30 Read block 376 len 16384 26/30 Write block 98 len 32768 data: 6 6 8 10 6 6 5 6 27/30 Read block 345 len 8192 28/30 Read block 105 len 40960 29/30 Read block 435 len 40960 30/30 Read block 548 len 20480 ---------------------------------------------------------------- Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Write 8 Done Done Done false 1002 Acked Read 2 Done Done Done false 1003 Acked Write 8 Done Done Done false 1004 Acked Write 3 Done Done Done false 1005 Acked Write 7 Done Done Done false 1006 Acked Read 3 Done Done Done false 1007 Acked Read 10 Done Done Done false 1008 Acked Read 6 Done Done Done false 1009 Acked Write 7 Done Sent Done false 1010 Acked Write 3 Done Sent Done false 1011 Acked Write 5 Done Sent Done false 1012 Acked Read 1 Done Sent Done false 1013 Acked Read 5 Done Sent Done false 1014 Acked Write 1 Done Sent Done false 1015 Acked Write 10 Done Sent Done false 1016 Acked Read 9 Done Sent Done false 1017 Acked Read 7 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Write 1 Done Sent Done false 1020 Acked Write 3 Done Sent Done false 1021 Acked Read 6 Done Sent Done false 1022 Acked Write 1 Done Sent Done false 1023 Acked Write 3 Done Sent Done false 1024 Acked Read 4 Done Sent Done false 1025 Acked Write 8 Done Sent Done false 1026 Acked Read 2 Done Sent Done false 1027 Acked Read 10 Done Sent Done false 1028 Acked Read 10 Done Sent Done false 1029 Acked Read 5 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 21 0 21 Done 30 9 30 69 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) None Some(JobId(1018)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:55:51.074867234Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1408} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:51.075063815Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1408,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 63 63 63 113 113 113 F F F 1 020-039 66 66 66 118 118 118 T F T <--- 2 040-059 68 66 68 120 118 120 F F F <--- 3 060-079 47 47 47 88 88 88 F F F 4 080-099 68 62 68 120 112 120 T T T <--- 5 100-119 68 65 68 120 116 120 T T T <--- 6 120-139 64 64 64 114 114 114 F F F 7 140-159 68 66 68 120 118 120 F F F <--- 8 160-179 68 64 68 120 115 120 F F F <--- 9 180-199 68 65 68 120 116 120 F F F <--- 10 200-219 65 65 65 116 116 116 F F F 11 220-239 66 66 66 117 117 117 T F T <--- 12 240-259 67 67 67 119 119 119 F F F 13 260-279 60 60 60 111 111 111 F F F 14 280-299 64 64 64 114 114 114 F F F 15 300-319 64 64 64 114 114 114 F F F 16 320-339 68 66 68 120 118 120 F T F <--- 17 340-359 66 66 66 117 117 117 F F F 18 360-379 67 67 67 119 119 119 F F F 19 380-399 64 64 64 114 114 114 F F F 20 400-419 66 66 66 117 117 117 F F F 21 420-439 68 62 68 120 112 120 F T F <--- 22 440-459 68 66 68 120 118 120 T T T <--- 23 460-479 65 65 65 116 116 116 F F F 24 480-499 65 65 65 116 116 116 F F F 25 500-519 68 65 68 120 116 120 F T F <--- 26 520-539 68 58 68 120 109 120 F T F <--- 27 540-559 55 55 55 104 104 104 F F F 28 560-579 60 60 60 111 111 111 F F F 29 580-599 68 66 68 120 117 120 F F F <--- Max gen: 68, Max flush: 120 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 34 Verifying data now No change after verify Loop: 34 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 63 63 63 113 113 113 F F F 1 020-039 69 69 69 121 121 121 F F F 2 040-059 68 68 68 120 120 120 F F F 3 060-079 47 47 47 88 88 88 F F F 4 080-099 69 69 69 121 121 121 F F F 5 100-119 69 69 69 121 121 121 F F F 6 120-139 64 64 64 114 114 114 F F F 7 140-159 68 68 68 120 120 120 F F F 8 160-179 68 68 68 120 120 120 F F F 9 180-199 68 68 68 120 120 120 F F F 10 200-219 65 65 65 116 116 116 F F F 11 220-239 69 69 69 121 121 121 F F F 12 240-259 67 67 67 119 119 119 F F F 13 260-279 60 60 60 111 111 111 F F F 14 280-299 64 64 64 114 114 114 F F F 15 300-319 64 64 64 114 114 114 F F F 16 320-339 68 68 68 120 120 120 F F F 17 340-359 66 66 66 117 117 117 F F F 18 360-379 67 67 67 119 119 119 F F F 19 380-399 64 64 64 114 114 114 F F F 20 400-419 66 66 66 117 117 117 F F F 21 420-439 68 68 68 120 120 120 F F F 22 440-459 69 69 69 121 121 121 F F F 23 460-479 65 65 65 116 116 116 F F F 24 480-499 65 65 65 116 116 116 F F F 25 500-519 68 68 68 120 120 120 F F F 26 520-539 68 68 68 120 120 120 F F F 27 540-559 55 55 55 104 104 104 F F F 28 560-579 60 60 60 111 111 111 F F F 29 580-599 68 68 68 120 120 120 F F F Max gen: 69, Max flush: 121 Begin loop 35 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:55.326051183Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1416,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:55.334024587Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1416,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:55.334316674Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1416} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:55.33434171Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1416} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:55.334350609Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1416} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:55.334362217Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1416} {"msg":"Crucible 4b4474ea-aa74-4074-a314-814e98e224f7 has session id: 50096784-1b35-447e-b0d8-1a472d58d052","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:55.334652894Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1416,"session_id":"50096784-1b35-447e-b0d8-1a472d58d052"} {"msg":"Upstairs opts: Upstairs UUID: 4b4474ea-aa74-4074-a314-814e98e224f7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:55.334668612Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1416,"session_id":"50096784-1b35-447e-b0d8-1a472d58d052"} {"msg":"Crucible stats registered with UUID: 4b4474ea-aa74-4074-a314-814e98e224f7","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:55.3346789Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1416,"session_id":"50096784-1b35-447e-b0d8-1a472d58d052"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:55:55.334694328Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1416,"session_id":"50096784-1b35-447e-b0d8-1a472d58d052"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:55.334835857Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1416,"":"io task","client":"0","":"downstairs","session_id":"50096784-1b35-447e-b0d8-1a472d58d052"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:55.334851965Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1416,"":"io task","client":"2","":"downstairs","session_id":"50096784-1b35-447e-b0d8-1a472d58d052"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:55.334862153Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1416,"":"io task","client":"1","":"downstairs","session_id":"50096784-1b35-447e-b0d8-1a472d58d052"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:55.335031628Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1416,"task":"crutest"} {"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:55.335043436Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1416} {"msg":"4b4474ea-aa74-4074-a314-814e98e224f7 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:55.335055035Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1416,"session_id":"50096784-1b35-447e-b0d8-1a472d58d052"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:55.335064043Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1416,"client":"0","":"downstairs","session_id":"50096784-1b35-447e-b0d8-1a472d58d052"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:55.335073912Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1416,"client":"1","":"downstairs","session_id":"50096784-1b35-447e-b0d8-1a472d58d052"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:55.33508367Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1416,"client":"2","":"downstairs","session_id":"50096784-1b35-447e-b0d8-1a472d58d052"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:55.335093399Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1416,"":"io task","client":"2","":"downstairs","session_id":"50096784-1b35-447e-b0d8-1a472d58d052"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:55.335103577Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1416,"":"io task","client":"0","":"downstairs","session_id":"50096784-1b35-447e-b0d8-1a472d58d052"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:55.335113716Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1416,"":"io task","client":"1","":"downstairs","session_id":"50096784-1b35-447e-b0d8-1a472d58d052"} {"msg":"ds_connection connected from Ok(127.0.0.1:58425)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:55.335161819Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1416,"":"io task","client":"2","":"downstairs","session_id":"50096784-1b35-447e-b0d8-1a472d58d052"} {"msg":"ds_connection connected from Ok(127.0.0.1:36853)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:55.335173007Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1416,"":"io task","client":"1","":"downstairs","session_id":"50096784-1b35-447e-b0d8-1a472d58d052"} {"msg":"ds_connection connected from Ok(127.0.0.1:49593)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:55.335183506Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1416,"":"io task","client":"0","":"downstairs","session_id":"50096784-1b35-447e-b0d8-1a472d58d052"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:55.335872934Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1416,"client":"2","":"downstairs","session_id":"50096784-1b35-447e-b0d8-1a472d58d052"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:55.335889821Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1416,"client":"0","":"downstairs","session_id":"50096784-1b35-447e-b0d8-1a472d58d052"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:55.335909369Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1416,"client":"1","":"downstairs","session_id":"50096784-1b35-447e-b0d8-1a472d58d052"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:55.336185648Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1416,"session_id":"50096784-1b35-447e-b0d8-1a472d58d052"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:55.336209654Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1416,"session_id":"50096784-1b35-447e-b0d8-1a472d58d052"} {"msg":"[0]R flush_numbers[0..12]: [113, 121, 120, 88, 121, 121, 114, 120, 120, 120, 116, 121]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:55.336261447Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1416,"":"downstairs","":"downstairs","session_id":"50096784-1b35-447e-b0d8-1a472d58d052"} {"msg":"[0]R generation[0..12]: [63, 69, 68, 47, 69, 69, 64, 68, 68, 68, 65, 69]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:55.336273335Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1416,"":"downstairs","":"downstairs","session_id":"50096784-1b35-447e-b0d8-1a472d58d052"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:55.336284453Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1416,"":"downstairs","":"downstairs","session_id":"50096784-1b35-447e-b0d8-1a472d58d052"} {"msg":"[1]R flush_numbers[0..12]: [113, 121, 120, 88, 121, 121, 114, 120, 120, 120, 116, 121]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:55.336295182Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1416,"":"downstairs","":"downstairs","session_id":"50096784-1b35-447e-b0d8-1a472d58d052"} {"msg":"[1]R generation[0..12]: [63, 69, 68, 47, 69, 69, 64, 68, 68, 68, 65, 69]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:55.33630565Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1416,"":"downstairs","":"downstairs","session_id":"50096784-1b35-447e-b0d8-1a472d58d052"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:55.336316978Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1416,"":"downstairs","":"downstairs","session_id":"50096784-1b35-447e-b0d8-1a472d58d052"} {"msg":"[2]R flush_numbers[0..12]: [113, 121, 120, 88, 121, 121, 114, 120, 120, 120, 116, 121]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:55.336327707Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1416,"":"downstairs","":"downstairs","session_id":"50096784-1b35-447e-b0d8-1a472d58d052"} {"msg":"[2]R generation[0..12]: [63, 69, 68, 47, 69, 69, 64, 68, 68, 68, 65, 69]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:55.336338105Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1416,"":"downstairs","":"downstairs","session_id":"50096784-1b35-447e-b0d8-1a472d58d052"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:55.336349044Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1416,"":"downstairs","":"downstairs","session_id":"50096784-1b35-447e-b0d8-1a472d58d052"} {"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:55.336358592Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1416,"":"downstairs","":"downstairs","session_id":"50096784-1b35-447e-b0d8-1a472d58d052"} {"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:55.336368311Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1416,"":"downstairs","":"downstairs","session_id":"50096784-1b35-447e-b0d8-1a472d58d052"} {"msg":"Next flush: 122","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:55.336377699Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1416,"":"downstairs","":"downstairs","session_id":"50096784-1b35-447e-b0d8-1a472d58d052"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:55.336387248Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1416,"":"downstairs","":"downstairs","session_id":"50096784-1b35-447e-b0d8-1a472d58d052"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:55.336399616Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1416,"session_id":"50096784-1b35-447e-b0d8-1a472d58d052"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:55.336408435Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1416,"client":"0","":"downstairs","session_id":"50096784-1b35-447e-b0d8-1a472d58d052"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:55.336418163Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1416,"client":"1","":"downstairs","session_id":"50096784-1b35-447e-b0d8-1a472d58d052"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:55.336427672Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1416,"client":"2","":"downstairs","session_id":"50096784-1b35-447e-b0d8-1a472d58d052"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:55.336437531Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1416,"session_id":"50096784-1b35-447e-b0d8-1a472d58d052"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:55.336446599Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1416,"session_id":"50096784-1b35-447e-b0d8-1a472d58d052"} {"msg":"4b4474ea-aa74-4074-a314-814e98e224f7 is now active with session: 50096784-1b35-447e-b0d8-1a472d58d052","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:55.336456528Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1416,"session_id":"50096784-1b35-447e-b0d8-1a472d58d052"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:55.336465247Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1416,"session_id":"50096784-1b35-447e-b0d8-1a472d58d052"} {"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:55.336474245Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1416} {"msg":"Activated sub_volume 4b4474ea-aa74-4074-a314-814e98e224f7","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:55.336482164Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1416} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:55.336490193Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1416,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:55.336810606Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1416,"task":"crutest"} 01/30 Write block 336 len 32768 data: 6 7 7 8 6 5 3 4 02/30 Write block 534 len 36864 data: 6 6 6 4 5 5 5 5 5 03/30 Read block 364 len 40960 04/30 Read block 42 len 32768 05/30 Write block 100 len 24576 data: 9 11 7 7 6 7 06/30 Write block 464 len 36864 data: 7 7 9 8 9 9 8 10 10 07/30 Write block 370 len 32768 data: 7 6 7 6 6 6 6 6 08/30 Read block 242 len 28672 09/30 Flush 10/30 Write block 432 len 40960 data: 7 6 7 8 7 8 9 7 5 6 11/30 Write block 35 len 28672 data: 8 8 9 13 13 12 12 12/30 Write block 140 len 24576 data: 9 9 8 7 8 8 13/30 Read block 418 len 8192 14/30 Read block 323 len 20480 15/30 Read block 435 len 8192 16/30 Write block 518 len 20480 data: 10 10 8 7 6 17/30 Read block 377 len 4096 18/30 Write block 197 len 16384 data: 8 8 7 5 19/30 Read block 12 len 12288 20/30 Read block 515 len 16384 21/30 Write block 466 len 16384 data: 10 9 10 10 22/30 Write block 311 len 12288 data: 9 8 8 23/30 Flush 24/30 Write block 592 len 20480 data: 9 10 9 8 6 25/30 Write block 582 len 28672 data: 8 8 9 9 9 9 5 26/30 Read block 440 len 24576 27/30 Write block 282 len 12288 data: 6 6 6 28/30 Read block 471 len 12288 29/30 Write block 186 len 16384 data: 3 5 4 6 30/30 Read block 194 len 20480 ---------------------------------------------------------------- Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Write 9 Done Done Done false 1002 Acked Read 10 Done Sent Done false 1003 Acked Read 8 Done Sent Done false 1004 Acked Write 6 Done Sent Done false 1005 Acked Write 9 Done Sent Done false 1006 Acked Write 8 Done Sent Done false 1007 Acked Read 7 Done Sent Done false 1008 Acked Flush 0 Done Sent Done false 1009 Acked Write 10 Done Sent Done false 1010 Acked Write 7 Done Sent Done false 1011 Acked Write 6 Done Sent Done false 1012 Acked Read 2 Done Sent Done false 1013 Acked Read 5 Done Sent Done false 1014 Acked Read 2 Done Sent Done false 1015 Acked Write 5 Done Sent Done false 1016 Acked Read 1 Done Sent Done false 1017 Acked Write 4 Done Sent Done false 1018 Acked Read 3 Done Sent Done false 1019 Acked Read 4 Done Sent Done false 1020 Acked Write 4 Done Sent Done false 1021 Acked Write 3 Done Sent Done false 1022 Acked Flush 0 Done Sent Done false 1023 Acked Write 5 Done Sent Done false 1024 Acked Write 7 Done Sent Done false 1025 Acked Read 6 Done Sent Done false 1026 Acked Write 3 Done Sent Done false 1027 Acked Read 3 Done Sent Done false 1028 Acked Write 4 Done Sent Done false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 28 1 29 Done 30 2 29 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) None Some(JobId(1022)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:55:55.34552242Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1416} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:55.345695745Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1416,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 63 63 63 113 113 113 F F F 1 020-039 70 69 70 123 121 123 F F F <--- 2 040-059 70 68 70 123 120 123 F F F <--- 3 060-079 47 47 47 88 88 88 F F F 4 080-099 69 69 69 121 121 121 F F F 5 100-119 70 69 70 122 121 122 F F F <--- 6 120-139 64 64 64 114 114 114 F F F 7 140-159 70 68 70 123 120 123 F F F <--- 8 160-179 68 68 68 120 120 120 F F F 9 180-199 70 68 70 123 120 123 T F T <--- 10 200-219 70 65 70 123 116 123 F F F <--- 11 220-239 69 69 69 121 121 121 F F F 12 240-259 67 67 67 119 119 119 F F F 13 260-279 60 60 60 111 111 111 F F F 14 280-299 64 64 64 114 114 114 T F T <--- 15 300-319 70 64 70 123 114 123 F F F <--- 16 320-339 70 68 70 122 120 122 F T F <--- 17 340-359 70 66 70 122 117 122 F T F <--- 18 360-379 70 67 70 122 119 122 F F F <--- 19 380-399 64 64 64 114 114 114 F F F 20 400-419 66 66 66 117 117 117 F F F 21 420-439 70 68 70 123 120 123 F F F <--- 22 440-459 70 69 70 123 121 123 F F F <--- 23 460-479 70 65 70 123 116 123 F F F <--- 24 480-499 65 65 65 116 116 116 F F F 25 500-519 70 68 70 123 120 123 F F F <--- 26 520-539 70 68 70 123 120 123 F T F <--- 27 540-559 70 55 70 122 104 122 F T F <--- 28 560-579 60 60 60 111 111 111 F F F 29 580-599 68 68 68 120 120 120 T F T <--- Max gen: 70, Max flush: 123 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 35 Verifying data now No change after verify Loop: 35 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 63 63 63 113 113 113 F F F 1 020-039 70 70 70 123 123 123 F F F 2 040-059 70 70 70 123 123 123 F F F 3 060-079 47 47 47 88 88 88 F F F 4 080-099 69 69 69 121 121 121 F F F 5 100-119 70 70 70 122 122 122 F F F 6 120-139 64 64 64 114 114 114 F F F 7 140-159 70 70 70 123 123 123 F F F 8 160-179 68 68 68 120 120 120 F F F 9 180-199 71 71 71 124 124 124 F F F 10 200-219 70 70 70 123 123 123 F F F 11 220-239 69 69 69 121 121 121 F F F 12 240-259 67 67 67 119 119 119 F F F 13 260-279 60 60 60 111 111 111 F F F 14 280-299 71 71 71 124 124 124 F F F 15 300-319 70 70 70 123 123 123 F F F 16 320-339 70 70 70 122 122 122 F F F 17 340-359 70 70 70 122 122 122 F F F 18 360-379 70 70 70 122 122 122 F F F 19 380-399 64 64 64 114 114 114 F F F 20 400-419 66 66 66 117 117 117 F F F 21 420-439 70 70 70 123 123 123 F F F 22 440-459 70 70 70 123 123 123 F F F 23 460-479 70 70 70 123 123 123 F F F 24 480-499 65 65 65 116 116 116 F F F 25 500-519 70 70 70 123 123 123 F F F 26 520-539 70 70 70 123 123 123 F F F 27 540-559 70 70 70 122 122 122 F F F 28 560-579 60 60 60 111 111 111 F F F 29 580-599 71 71 71 124 124 124 F F F Max gen: 71, Max flush: 124 Begin loop 36 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:59.601551424Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1424,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:59.609665497Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1424,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:59.609959024Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1424} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:59.609993119Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1424} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:59.610023924Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1424} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:59.610035843Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1424} {"msg":"Crucible fcddec33-765a-44b9-b72c-b2f48c404450 has session id: 6240be7a-d0bf-4ba1-a6cd-3a9123da7914","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:59.610383671Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1424,"session_id":"6240be7a-d0bf-4ba1-a6cd-3a9123da7914"} {"msg":"Upstairs opts: Upstairs UUID: fcddec33-765a-44b9-b72c-b2f48c404450, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:59.610413187Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1424,"session_id":"6240be7a-d0bf-4ba1-a6cd-3a9123da7914"} {"msg":"Crucible stats registered with UUID: fcddec33-765a-44b9-b72c-b2f48c404450","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:59.610424415Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1424,"session_id":"6240be7a-d0bf-4ba1-a6cd-3a9123da7914"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:55:59.610433714Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1424,"session_id":"6240be7a-d0bf-4ba1-a6cd-3a9123da7914"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:59.610576713Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1424,"":"io task","client":"1","":"downstairs","session_id":"6240be7a-d0bf-4ba1-a6cd-3a9123da7914"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:59.610608698Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1424,"":"io task","client":"0","":"downstairs","session_id":"6240be7a-d0bf-4ba1-a6cd-3a9123da7914"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:59.610619697Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1424,"":"io task","client":"2","":"downstairs","session_id":"6240be7a-d0bf-4ba1-a6cd-3a9123da7914"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:59.610740509Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1424,"task":"crutest"} {"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:59.610754837Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1424} {"msg":"fcddec33-765a-44b9-b72c-b2f48c404450 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:59.610793001Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1424,"session_id":"6240be7a-d0bf-4ba1-a6cd-3a9123da7914"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:59.61080271Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1424,"client":"0","":"downstairs","session_id":"6240be7a-d0bf-4ba1-a6cd-3a9123da7914"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:59.610812888Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1424,"client":"1","":"downstairs","session_id":"6240be7a-d0bf-4ba1-a6cd-3a9123da7914"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:59.610825396Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1424,"client":"2","":"downstairs","session_id":"6240be7a-d0bf-4ba1-a6cd-3a9123da7914"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:59.610835335Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1424,"":"io task","client":"2","":"downstairs","session_id":"6240be7a-d0bf-4ba1-a6cd-3a9123da7914"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:59.610845423Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1424,"":"io task","client":"0","":"downstairs","session_id":"6240be7a-d0bf-4ba1-a6cd-3a9123da7914"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:59.610877119Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1424,"":"io task","client":"1","":"downstairs","session_id":"6240be7a-d0bf-4ba1-a6cd-3a9123da7914"} {"msg":"ds_connection connected from Ok(127.0.0.1:50794)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:59.610901055Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1424,"":"io task","client":"1","":"downstairs","session_id":"6240be7a-d0bf-4ba1-a6cd-3a9123da7914"} {"msg":"ds_connection connected from Ok(127.0.0.1:39154)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:59.610929441Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1424,"":"io task","client":"2","":"downstairs","session_id":"6240be7a-d0bf-4ba1-a6cd-3a9123da7914"} {"msg":"ds_connection connected from Ok(127.0.0.1:54641)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:59.610954217Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1424,"":"io task","client":"0","":"downstairs","session_id":"6240be7a-d0bf-4ba1-a6cd-3a9123da7914"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:59.611619029Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1424,"client":"1","":"downstairs","session_id":"6240be7a-d0bf-4ba1-a6cd-3a9123da7914"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:59.611652844Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1424,"client":"2","":"downstairs","session_id":"6240be7a-d0bf-4ba1-a6cd-3a9123da7914"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:59.611664382Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1424,"client":"0","":"downstairs","session_id":"6240be7a-d0bf-4ba1-a6cd-3a9123da7914"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:59.611932273Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1424,"session_id":"6240be7a-d0bf-4ba1-a6cd-3a9123da7914"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:59.611941891Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1424,"session_id":"6240be7a-d0bf-4ba1-a6cd-3a9123da7914"} {"msg":"[0]R flush_numbers[0..12]: [113, 123, 123, 88, 121, 122, 114, 123, 120, 124, 123, 121]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:59.612049536Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1424,"":"downstairs","":"downstairs","session_id":"6240be7a-d0bf-4ba1-a6cd-3a9123da7914"} {"msg":"[0]R generation[0..12]: [63, 70, 70, 47, 69, 70, 64, 70, 68, 71, 70, 69]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:59.612060724Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1424,"":"downstairs","":"downstairs","session_id":"6240be7a-d0bf-4ba1-a6cd-3a9123da7914"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:59.612071972Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1424,"":"downstairs","":"downstairs","session_id":"6240be7a-d0bf-4ba1-a6cd-3a9123da7914"} {"msg":"[1]R flush_numbers[0..12]: [113, 123, 123, 88, 121, 122, 114, 123, 120, 124, 123, 121]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:59.612085Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1424,"":"downstairs","":"downstairs","session_id":"6240be7a-d0bf-4ba1-a6cd-3a9123da7914"} {"msg":"[1]R generation[0..12]: [63, 70, 70, 47, 69, 70, 64, 70, 68, 71, 70, 69]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:59.612095959Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1424,"":"downstairs","":"downstairs","session_id":"6240be7a-d0bf-4ba1-a6cd-3a9123da7914"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:59.612107147Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1424,"":"downstairs","":"downstairs","session_id":"6240be7a-d0bf-4ba1-a6cd-3a9123da7914"} {"msg":"[2]R flush_numbers[0..12]: [113, 123, 123, 88, 121, 122, 114, 123, 120, 124, 123, 121]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:59.612117656Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1424,"":"downstairs","":"downstairs","session_id":"6240be7a-d0bf-4ba1-a6cd-3a9123da7914"} {"msg":"[2]R generation[0..12]: [63, 70, 70, 47, 69, 70, 64, 70, 68, 71, 70, 69]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:59.612128064Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1424,"":"downstairs","":"downstairs","session_id":"6240be7a-d0bf-4ba1-a6cd-3a9123da7914"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:59.612139182Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1424,"":"downstairs","":"downstairs","session_id":"6240be7a-d0bf-4ba1-a6cd-3a9123da7914"} {"msg":"Max found gen is 72","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:59.612148711Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1424,"":"downstairs","":"downstairs","session_id":"6240be7a-d0bf-4ba1-a6cd-3a9123da7914"} {"msg":"Generation requested: 72 >= found:72","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:59.61215847Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1424,"":"downstairs","":"downstairs","session_id":"6240be7a-d0bf-4ba1-a6cd-3a9123da7914"} {"msg":"Next flush: 125","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:59.612168268Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1424,"":"downstairs","":"downstairs","session_id":"6240be7a-d0bf-4ba1-a6cd-3a9123da7914"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:59.612177877Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1424,"":"downstairs","":"downstairs","session_id":"6240be7a-d0bf-4ba1-a6cd-3a9123da7914"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:59.612187895Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1424,"session_id":"6240be7a-d0bf-4ba1-a6cd-3a9123da7914"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:59.612196634Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1424,"client":"0","":"downstairs","session_id":"6240be7a-d0bf-4ba1-a6cd-3a9123da7914"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:59.612206482Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1424,"client":"1","":"downstairs","session_id":"6240be7a-d0bf-4ba1-a6cd-3a9123da7914"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:59.612216231Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1424,"client":"2","":"downstairs","session_id":"6240be7a-d0bf-4ba1-a6cd-3a9123da7914"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:59.61222609Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1424,"session_id":"6240be7a-d0bf-4ba1-a6cd-3a9123da7914"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:59.612236508Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1424,"session_id":"6240be7a-d0bf-4ba1-a6cd-3a9123da7914"} {"msg":"fcddec33-765a-44b9-b72c-b2f48c404450 is now active with session: 6240be7a-d0bf-4ba1-a6cd-3a9123da7914","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:59.612246437Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1424,"session_id":"6240be7a-d0bf-4ba1-a6cd-3a9123da7914"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:59.612255335Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1424,"session_id":"6240be7a-d0bf-4ba1-a6cd-3a9123da7914"} {"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:59.612264494Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1424} {"msg":"Activated sub_volume fcddec33-765a-44b9-b72c-b2f48c404450","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:59.612272493Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1424} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:59.612280551Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1424,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:59.612540843Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1424,"task":"crutest"} 01/30 Write block 121 len 20480 data: 5 4 5 2 4 02/30 Read block 464 len 32768 03/30 Read block 373 len 4096 04/30 Read block 25 len 4096 05/30 Write block 146 len 12288 data: 7 8 8 06/30 Read block 194 len 24576 07/30 Write block 315 len 12288 data: 8 8 6 08/30 Write block 209 len 16384 data: 7 5 6 7 09/30 Write block 248 len 28672 data: 8 6 7 6 6 6 6 10/30 Read block 42 len 12288 11/30 Read block 594 len 4096 12/30 Read block 554 len 8192 13/30 Write block 119 len 24576 data: 6 5 6 5 6 3 14/30 Write block 5 len 8192 data: 5 5 15/30 Write block 180 len 12288 data: 5 6 5 16/30 Read block 309 len 20480 17/30 Write block 511 len 24576 data: 5 6 8 8 9 7 18/30 Flush 19/30 Write block 56 len 12288 data: 5 3 3 20/30 Read block 398 len 12288 21/30 Read block 320 len 40960 22/30 Write block 308 len 24576 data: 4 6 7 10 9 9 23/30 Read block 137 len 16384 24/30 Read block 554 len 32768 25/30 Write block 182 len 4096 data: 6 26/30 Read block 13 len 16384 27/30 Write block 527 len 8192 data: 3 4 28/30 Write block 506 len 28672 data: 2 3 2 2 6 6 7 29/30 Read block 399 len 40960 30/30 Write block 479 len 16384 data: 8 8 7 7 ---------------------------------------------------------------- Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Read 8 Done Done Done false 1002 Acked Read 1 Done Sent Done false 1003 Acked Read 1 Done Sent Done false 1004 Acked Write 3 Done Sent Done false 1005 Acked Read 6 Done Sent Done false 1006 Acked Write 3 Done Sent Done false 1007 Acked Write 4 Done Sent Done false 1008 Acked Write 7 Done Sent Done false 1009 Acked Read 3 Done Sent Done false 1010 Acked Read 1 Done Sent Done false 1011 Acked Read 2 Done Sent Done false 1012 Acked Write 6 Done Sent Done false 1013 Acked Write 2 Done Sent Done false 1014 Acked Write 3 Done Sent Done false 1015 Acked Read 5 Done Sent Done false 1016 Acked Write 6 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Write 3 Done Sent Done false 1019 Acked Read 3 Done Sent Done false 1020 Acked Read 10 Done Sent Done false 1021 Acked Write 6 Done Sent Done false 1022 Acked Read 4 Done Sent Done false 1023 Acked Read 8 Done Sent Done false 1024 Acked Write 1 Done Sent Done false 1025 Acked Read 4 Done Sent Done false 1026 Acked Write 2 Done Sent Done false 1027 Acked Write 7 Done Sent Done false 1028 Acked Read 10 Done Sent Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 28 1 30 Done 29 2 29 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) None Some(JobId(1017)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:55:59.620803174Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1424} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:55:59.621002275Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1424,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 72 63 72 125 113 125 F F F <--- 1 020-039 70 70 70 123 123 123 F F F 2 040-059 70 70 70 123 123 123 T F T <--- 3 060-079 47 47 47 88 88 88 F F F 4 080-099 69 69 69 121 121 121 F F F 5 100-119 72 70 72 125 122 125 F F F <--- 6 120-139 72 64 72 125 114 125 F T F <--- 7 140-159 72 70 72 125 123 125 F F F <--- 8 160-179 68 68 68 120 120 120 F F F 9 180-199 72 71 72 125 124 125 T F T <--- 10 200-219 72 70 72 125 123 125 F F F <--- 11 220-239 69 69 69 121 121 121 F F F 12 240-259 72 67 72 125 119 125 F F F <--- 13 260-279 60 60 60 111 111 111 F F F 14 280-299 71 71 71 124 124 124 F F F 15 300-319 72 70 72 125 123 125 T F T <--- 16 320-339 70 70 70 122 122 122 F F F 17 340-359 70 70 70 122 122 122 F F F 18 360-379 70 70 70 122 122 122 F F F 19 380-399 64 64 64 114 114 114 F F F 20 400-419 66 66 66 117 117 117 F F F 21 420-439 70 70 70 123 123 123 F F F 22 440-459 70 70 70 123 123 123 F F F 23 460-479 70 70 70 123 123 123 T F T <--- 24 480-499 65 65 65 116 116 116 T F T <--- 25 500-519 72 70 72 125 123 125 T F T <--- 26 520-539 70 70 70 123 123 123 T F T <--- 27 540-559 70 70 70 122 122 122 F F F 28 560-579 60 60 60 111 111 111 F F F 29 580-599 71 71 71 124 124 124 F F F Max gen: 72, Max flush: 125 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 36 Verifying data now No change after verify Loop: 36 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 72 72 72 125 125 125 F F F 1 020-039 70 70 70 123 123 123 F F F 2 040-059 73 73 73 126 126 126 F F F 3 060-079 47 47 47 88 88 88 F F F 4 080-099 69 69 69 121 121 121 F F F 5 100-119 72 72 72 125 125 125 F F F 6 120-139 72 72 72 125 125 125 F F F 7 140-159 72 72 72 125 125 125 F F F 8 160-179 68 68 68 120 120 120 F F F 9 180-199 73 73 73 126 126 126 F F F 10 200-219 72 72 72 125 125 125 F F F 11 220-239 69 69 69 121 121 121 F F F 12 240-259 72 72 72 125 125 125 F F F 13 260-279 60 60 60 111 111 111 F F F 14 280-299 71 71 71 124 124 124 F F F 15 300-319 73 73 73 126 126 126 F F F 16 320-339 70 70 70 122 122 122 F F F 17 340-359 70 70 70 122 122 122 F F F 18 360-379 70 70 70 122 122 122 F F F 19 380-399 64 64 64 114 114 114 F F F 20 400-419 66 66 66 117 117 117 F F F 21 420-439 70 70 70 123 123 123 F F F 22 440-459 70 70 70 123 123 123 F F F 23 460-479 73 73 73 126 126 126 F F F 24 480-499 73 73 73 126 126 126 F F F 25 500-519 73 73 73 126 126 126 F F F 26 520-539 73 73 73 126 126 126 F F F 27 540-559 70 70 70 122 122 122 F F F 28 560-579 60 60 60 111 111 111 F F F 29 580-599 71 71 71 124 124 124 F F F Max gen: 73, Max flush: 126 Begin loop 37 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:03.874935067Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1432,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:03.88318642Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1432,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:03.883510492Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1432} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:03.883537048Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1432} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:03.883545627Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1432} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:03.883557255Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1432} {"msg":"Crucible 0d3ace0b-ac8b-433c-a28a-fb30ab915c40 has session id: bc2dd829-a041-4217-a3fc-fcaa25472791","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:03.883928621Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1432,"session_id":"bc2dd829-a041-4217-a3fc-fcaa25472791"} {"msg":"Upstairs opts: Upstairs UUID: 0d3ace0b-ac8b-433c-a28a-fb30ab915c40, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:03.883959096Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1432,"session_id":"bc2dd829-a041-4217-a3fc-fcaa25472791"} {"msg":"Crucible stats registered with UUID: 0d3ace0b-ac8b-433c-a28a-fb30ab915c40","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:03.883973104Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1432,"session_id":"bc2dd829-a041-4217-a3fc-fcaa25472791"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:56:03.883982523Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1432,"session_id":"bc2dd829-a041-4217-a3fc-fcaa25472791"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:03.884109944Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1432,"":"io task","client":"2","":"downstairs","session_id":"bc2dd829-a041-4217-a3fc-fcaa25472791"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:03.884126791Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1432,"":"io task","client":"0","":"downstairs","session_id":"bc2dd829-a041-4217-a3fc-fcaa25472791"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:03.88413701Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1432,"":"io task","client":"1","":"downstairs","session_id":"bc2dd829-a041-4217-a3fc-fcaa25472791"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:03.884311554Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1432,"task":"crutest"} {"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:03.884326572Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1432} {"msg":"0d3ace0b-ac8b-433c-a28a-fb30ab915c40 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:03.884343539Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1432,"session_id":"bc2dd829-a041-4217-a3fc-fcaa25472791"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:03.884352378Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1432,"client":"0","":"downstairs","session_id":"bc2dd829-a041-4217-a3fc-fcaa25472791"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:03.884362277Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1432,"client":"1","":"downstairs","session_id":"bc2dd829-a041-4217-a3fc-fcaa25472791"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:03.884371855Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1432,"client":"2","":"downstairs","session_id":"bc2dd829-a041-4217-a3fc-fcaa25472791"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:03.884381444Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1432,"":"io task","client":"2","":"downstairs","session_id":"bc2dd829-a041-4217-a3fc-fcaa25472791"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:03.884391572Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1432,"":"io task","client":"0","":"downstairs","session_id":"bc2dd829-a041-4217-a3fc-fcaa25472791"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:03.884401851Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1432,"":"io task","client":"1","":"downstairs","session_id":"bc2dd829-a041-4217-a3fc-fcaa25472791"} {"msg":"ds_connection connected from Ok(127.0.0.1:57941)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:03.884499886Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1432,"":"io task","client":"2","":"downstairs","session_id":"bc2dd829-a041-4217-a3fc-fcaa25472791"} {"msg":"ds_connection connected from Ok(127.0.0.1:55998)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:03.88460936Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1432,"":"io task","client":"1","":"downstairs","session_id":"bc2dd829-a041-4217-a3fc-fcaa25472791"} {"msg":"ds_connection connected from Ok(127.0.0.1:55804)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:03.884623248Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1432,"":"io task","client":"0","":"downstairs","session_id":"bc2dd829-a041-4217-a3fc-fcaa25472791"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:03.88515462Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1432,"client":"2","":"downstairs","session_id":"bc2dd829-a041-4217-a3fc-fcaa25472791"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:03.885176976Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1432,"client":"1","":"downstairs","session_id":"bc2dd829-a041-4217-a3fc-fcaa25472791"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:03.885188545Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1432,"client":"0","":"downstairs","session_id":"bc2dd829-a041-4217-a3fc-fcaa25472791"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:03.885542512Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1432,"session_id":"bc2dd829-a041-4217-a3fc-fcaa25472791"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:03.885589646Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1432,"session_id":"bc2dd829-a041-4217-a3fc-fcaa25472791"} {"msg":"[0]R flush_numbers[0..12]: [125, 123, 126, 88, 121, 125, 125, 125, 120, 126, 125, 121]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:03.885652426Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1432,"":"downstairs","":"downstairs","session_id":"bc2dd829-a041-4217-a3fc-fcaa25472791"} {"msg":"[0]R generation[0..12]: [72, 70, 73, 47, 69, 72, 72, 72, 68, 73, 72, 69]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:03.885663775Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1432,"":"downstairs","":"downstairs","session_id":"bc2dd829-a041-4217-a3fc-fcaa25472791"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:03.885675163Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1432,"":"downstairs","":"downstairs","session_id":"bc2dd829-a041-4217-a3fc-fcaa25472791"} {"msg":"[1]R flush_numbers[0..12]: [125, 123, 126, 88, 121, 125, 125, 125, 120, 126, 125, 121]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:03.885685641Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1432,"":"downstairs","":"downstairs","session_id":"bc2dd829-a041-4217-a3fc-fcaa25472791"} {"msg":"[1]R generation[0..12]: [72, 70, 73, 47, 69, 72, 72, 72, 68, 73, 72, 69]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:03.88569618Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1432,"":"downstairs","":"downstairs","session_id":"bc2dd829-a041-4217-a3fc-fcaa25472791"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:03.885707058Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1432,"":"downstairs","":"downstairs","session_id":"bc2dd829-a041-4217-a3fc-fcaa25472791"} {"msg":"[2]R flush_numbers[0..12]: [125, 123, 126, 88, 121, 125, 125, 125, 120, 126, 125, 121]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:03.885717487Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1432,"":"downstairs","":"downstairs","session_id":"bc2dd829-a041-4217-a3fc-fcaa25472791"} {"msg":"[2]R generation[0..12]: [72, 70, 73, 47, 69, 72, 72, 72, 68, 73, 72, 69]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:03.885727795Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1432,"":"downstairs","":"downstairs","session_id":"bc2dd829-a041-4217-a3fc-fcaa25472791"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:03.885741033Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1432,"":"downstairs","":"downstairs","session_id":"bc2dd829-a041-4217-a3fc-fcaa25472791"} {"msg":"Max found gen is 74","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:03.885750652Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1432,"":"downstairs","":"downstairs","session_id":"bc2dd829-a041-4217-a3fc-fcaa25472791"} {"msg":"Generation requested: 74 >= found:74","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:03.8857602Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1432,"":"downstairs","":"downstairs","session_id":"bc2dd829-a041-4217-a3fc-fcaa25472791"} {"msg":"Next flush: 127","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:03.885769939Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1432,"":"downstairs","":"downstairs","session_id":"bc2dd829-a041-4217-a3fc-fcaa25472791"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:03.885779178Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1432,"":"downstairs","":"downstairs","session_id":"bc2dd829-a041-4217-a3fc-fcaa25472791"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:03.885788996Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1432,"session_id":"bc2dd829-a041-4217-a3fc-fcaa25472791"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:03.885797805Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1432,"client":"0","":"downstairs","session_id":"bc2dd829-a041-4217-a3fc-fcaa25472791"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:03.885807593Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1432,"client":"1","":"downstairs","session_id":"bc2dd829-a041-4217-a3fc-fcaa25472791"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:03.885817242Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1432,"client":"2","":"downstairs","session_id":"bc2dd829-a041-4217-a3fc-fcaa25472791"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:03.88582712Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1432,"session_id":"bc2dd829-a041-4217-a3fc-fcaa25472791"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:03.885836329Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1432,"session_id":"bc2dd829-a041-4217-a3fc-fcaa25472791"} {"msg":"0d3ace0b-ac8b-433c-a28a-fb30ab915c40 is now active with session: bc2dd829-a041-4217-a3fc-fcaa25472791","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:03.885846268Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1432,"session_id":"bc2dd829-a041-4217-a3fc-fcaa25472791"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:03.885855106Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1432,"session_id":"bc2dd829-a041-4217-a3fc-fcaa25472791"} {"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:03.885864105Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1432} {"msg":"Activated sub_volume 0d3ace0b-ac8b-433c-a28a-fb30ab915c40","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:03.885872164Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1432} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:03.885880243Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1432,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:03.886252498Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1432,"task":"crutest"} 01/30 Write block 52 len 36864 data: 3 3 5 5 6 4 4 2 3 02/30 Write block 449 len 36864 data: 8 7 8 7 6 5 4 6 7 03/30 Read block 536 len 40960 04/30 Write block 253 len 28672 data: 7 7 7 7 7 8 6 05/30 Read block 110 len 12288 06/30 Read block 181 len 36864 07/30 Write block 385 len 12288 data: 5 6 6 08/30 Read block 444 len 32768 09/30 Write block 169 len 16384 data: 3 2 2 3 10/30 Write block 270 len 24576 data: 9 11 9 7 8 9 11/30 Write block 62 len 24576 data: 4 3 3 3 4 5 12/30 Write block 100 len 16384 data: 10 12 8 8 13/30 Write block 445 len 40960 data: 7 7 7 8 9 8 9 8 7 6 14/30 Read block 185 len 20480 15/30 Write block 492 len 8192 data: 5 4 16/30 Read block 255 len 28672 17/30 Read block 464 len 8192 18/30 Write block 76 len 36864 data: 5 5 5 5 4 2 3 3 5 19/30 Write block 397 len 16384 data: 5 4 4 4 20/30 Read block 44 len 12288 21/30 Read block 229 len 20480 22/30 Write block 206 len 24576 data: 7 11 9 8 6 7 23/30 Write block 450 len 20480 data: 9 10 9 8 7 24/30 Read block 257 len 8192 25/30 Write block 277 len 36864 data: 8 9 6 6 5 7 7 7 6 26/30 Flush 27/30 Write block 43 len 20480 data: 10 10 9 9 9 28/30 Read block 317 len 16384 29/30 Read block 468 len 24576 30/30 Read block 559 len 40960 ---------------------------------------------------------------- Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Sent Done false 1001 Acked Write 9 Done Sent Done false 1002 Acked Read 10 Done Sent Done false 1003 Acked Write 7 Done Sent Done false 1004 Acked Read 3 Done Sent Done false 1005 Acked Read 9 Done Sent Done false 1006 Acked Write 3 Done Sent Done false 1007 Acked Read 8 Done Sent Done false 1008 Acked Write 4 Done Sent Done false 1009 Acked Write 6 Done Sent Done false 1010 Acked Write 6 Done Sent Done false 1011 Acked Write 4 Done Sent Done false 1012 Acked Write 10 Done Sent Done false 1013 Acked Read 5 Done Sent Done false 1014 Acked Write 2 Done Sent Done false 1015 Acked Read 7 Done Sent Done false 1016 Acked Read 2 Done Sent Done false 1017 Acked Write 9 Done Sent Done false 1018 Acked Write 4 Done Sent Done false 1019 Acked Read 3 Done Sent Done false 1020 Acked Read 5 Done Sent Done false 1021 Acked Write 6 Done Sent Done false 1022 Acked Write 5 Done Sent Done false 1023 Acked Read 2 Done Sent Done false 1024 Acked Write 9 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 5 Done Sent Done false 1027 Acked Read 4 Done Sent Done false 1028 Acked Read 6 Done Sent Done false 1029 Acked Read 10 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 30 0 30 Done 30 0 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) None Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:56:03.895393119Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1432} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:03.895568453Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1432,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 72 72 72 125 125 125 F F F 1 020-039 70 70 70 123 123 123 F F F 2 040-059 74 73 74 127 126 127 T F T <--- 3 060-079 74 47 74 127 88 127 F F F <--- 4 080-099 74 69 74 127 121 127 F F F <--- 5 100-119 74 72 74 127 125 127 F F F <--- 6 120-139 72 72 72 125 125 125 F F F 7 140-159 72 72 72 125 125 125 F F F 8 160-179 74 68 74 127 120 127 F F F <--- 9 180-199 73 73 73 126 126 126 F F F 10 200-219 74 72 74 127 125 127 F F F <--- 11 220-239 69 69 69 121 121 121 F F F 12 240-259 74 72 74 127 125 127 F F F <--- 13 260-279 74 60 74 127 111 127 F F F <--- 14 280-299 74 71 74 127 124 127 F F F <--- 15 300-319 73 73 73 126 126 126 F F F 16 320-339 70 70 70 122 122 122 F F F 17 340-359 70 70 70 122 122 122 F F F 18 360-379 70 70 70 122 122 122 F F F 19 380-399 74 64 74 127 114 127 F F F <--- 20 400-419 74 66 74 127 117 127 F F F <--- 21 420-439 70 70 70 123 123 123 F F F 22 440-459 74 70 74 127 123 127 F F F <--- 23 460-479 73 73 73 126 126 126 F F F 24 480-499 74 73 74 127 126 127 F F F <--- 25 500-519 73 73 73 126 126 126 F F F 26 520-539 73 73 73 126 126 126 F F F 27 540-559 70 70 70 122 122 122 F F F 28 560-579 60 60 60 111 111 111 F F F 29 580-599 71 71 71 124 124 124 F F F Max gen: 74, Max flush: 127 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 37 Verifying data now No change after verify Loop: 37 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 72 72 72 125 125 125 F F F 1 020-039 70 70 70 123 123 123 F F F 2 040-059 75 75 75 128 128 128 F F F 3 060-079 74 74 74 127 127 127 F F F 4 080-099 74 74 74 127 127 127 F F F 5 100-119 74 74 74 127 127 127 F F F 6 120-139 72 72 72 125 125 125 F F F 7 140-159 72 72 72 125 125 125 F F F 8 160-179 74 74 74 127 127 127 F F F 9 180-199 73 73 73 126 126 126 F F F 10 200-219 74 74 74 127 127 127 F F F 11 220-239 69 69 69 121 121 121 F F F 12 240-259 74 74 74 127 127 127 F F F 13 260-279 74 74 74 127 127 127 F F F 14 280-299 74 74 74 127 127 127 F F F 15 300-319 73 73 73 126 126 126 F F F 16 320-339 70 70 70 122 122 122 F F F 17 340-359 70 70 70 122 122 122 F F F 18 360-379 70 70 70 122 122 122 F F F 19 380-399 74 74 74 127 127 127 F F F 20 400-419 74 74 74 127 127 127 F F F 21 420-439 70 70 70 123 123 123 F F F 22 440-459 74 74 74 127 127 127 F F F 23 460-479 73 73 73 126 126 126 F F F 24 480-499 74 74 74 127 127 127 F F F 25 500-519 73 73 73 126 126 126 F F F 26 520-539 73 73 73 126 126 126 F F F 27 540-559 70 70 70 122 122 122 F F F 28 560-579 60 60 60 111 111 111 F F F 29 580-599 71 71 71 124 124 124 F F F Max gen: 75, Max flush: 128 Begin loop 38 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:08.143342535Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1440,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:56:08.150467244Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1440,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:08.150528915Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1440,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:08.157622798Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1440,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:08.157896558Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1440} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:08.157921614Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1440} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:08.157930113Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1440} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:08.157941711Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1440} {"msg":"Crucible 614384c0-c658-482d-a988-b6eef262afaf has session id: a000f6ae-743e-4ce2-af99-e1850a1f88e6","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:08.158315456Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1440,"session_id":"a000f6ae-743e-4ce2-af99-e1850a1f88e6"} {"msg":"Upstairs opts: Upstairs UUID: 614384c0-c658-482d-a988-b6eef262afaf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:08.158338373Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1440,"session_id":"a000f6ae-743e-4ce2-af99-e1850a1f88e6"} {"msg":"Crucible stats registered with UUID: 614384c0-c658-482d-a988-b6eef262afaf","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:08.158348621Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1440,"session_id":"a000f6ae-743e-4ce2-af99-e1850a1f88e6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:56:08.15835798Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1440,"session_id":"a000f6ae-743e-4ce2-af99-e1850a1f88e6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:08.158505558Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1440,"":"io task","client":"0","":"downstairs","session_id":"a000f6ae-743e-4ce2-af99-e1850a1f88e6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:08.158537634Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1440,"":"io task","client":"1","":"downstairs","session_id":"a000f6ae-743e-4ce2-af99-e1850a1f88e6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:08.158548942Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1440,"":"io task","client":"2","":"downstairs","session_id":"a000f6ae-743e-4ce2-af99-e1850a1f88e6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:08.158693251Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1440,"task":"crutest"} {"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:08.158711838Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1440} {"msg":"614384c0-c658-482d-a988-b6eef262afaf active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:08.158736364Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1440,"session_id":"a000f6ae-743e-4ce2-af99-e1850a1f88e6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:08.158745843Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1440,"client":"0","":"downstairs","session_id":"a000f6ae-743e-4ce2-af99-e1850a1f88e6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:08.158755771Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1440,"client":"1","":"downstairs","session_id":"a000f6ae-743e-4ce2-af99-e1850a1f88e6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:08.15876551Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1440,"client":"2","":"downstairs","session_id":"a000f6ae-743e-4ce2-af99-e1850a1f88e6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:08.158775458Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1440,"":"io task","client":"2","":"downstairs","session_id":"a000f6ae-743e-4ce2-af99-e1850a1f88e6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:08.158785627Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1440,"":"io task","client":"0","":"downstairs","session_id":"a000f6ae-743e-4ce2-af99-e1850a1f88e6"} {"msg":"ds_connection connected from Ok(127.0.0.1:45727)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:08.158823511Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1440,"":"io task","client":"2","":"downstairs","session_id":"a000f6ae-743e-4ce2-af99-e1850a1f88e6"} {"msg":"ds_connection connected from Ok(127.0.0.1:44164)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:08.158844688Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1440,"":"io task","client":"0","":"downstairs","session_id":"a000f6ae-743e-4ce2-af99-e1850a1f88e6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:08.158855367Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1440,"":"io task","client":"1","":"downstairs","session_id":"a000f6ae-743e-4ce2-af99-e1850a1f88e6"} {"msg":"ds_connection connected from Ok(127.0.0.1:46501)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:08.158956692Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1440,"":"io task","client":"1","":"downstairs","session_id":"a000f6ae-743e-4ce2-af99-e1850a1f88e6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:08.15944142Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1440,"client":"1","":"downstairs","session_id":"a000f6ae-743e-4ce2-af99-e1850a1f88e6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:08.159468636Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1440,"client":"2","":"downstairs","session_id":"a000f6ae-743e-4ce2-af99-e1850a1f88e6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:08.159620534Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1440,"client":"0","":"downstairs","session_id":"a000f6ae-743e-4ce2-af99-e1850a1f88e6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:08.159742686Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1440,"session_id":"a000f6ae-743e-4ce2-af99-e1850a1f88e6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:08.159753644Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1440,"session_id":"a000f6ae-743e-4ce2-af99-e1850a1f88e6"} {"msg":"[0]R flush_numbers[0..12]: [125, 123, 128, 127, 127, 127, 125, 125, 127, 126, 127, 121]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:08.159897773Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1440,"":"downstairs","":"downstairs","session_id":"a000f6ae-743e-4ce2-af99-e1850a1f88e6"} {"msg":"[0]R generation[0..12]: [72, 70, 75, 74, 74, 74, 72, 72, 74, 73, 74, 69]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:08.159909181Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1440,"":"downstairs","":"downstairs","session_id":"a000f6ae-743e-4ce2-af99-e1850a1f88e6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:08.15992024Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1440,"":"downstairs","":"downstairs","session_id":"a000f6ae-743e-4ce2-af99-e1850a1f88e6"} {"msg":"[1]R flush_numbers[0..12]: [125, 123, 128, 127, 127, 127, 125, 125, 127, 126, 127, 121]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:08.159931188Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1440,"":"downstairs","":"downstairs","session_id":"a000f6ae-743e-4ce2-af99-e1850a1f88e6"} {"msg":"[1]R generation[0..12]: [72, 70, 75, 74, 74, 74, 72, 72, 74, 73, 74, 69]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:08.159941746Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1440,"":"downstairs","":"downstairs","session_id":"a000f6ae-743e-4ce2-af99-e1850a1f88e6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:08.159952845Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1440,"":"downstairs","":"downstairs","session_id":"a000f6ae-743e-4ce2-af99-e1850a1f88e6"} {"msg":"[2]R flush_numbers[0..12]: [125, 123, 128, 127, 127, 127, 125, 125, 127, 126, 127, 121]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:08.159963533Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1440,"":"downstairs","":"downstairs","session_id":"a000f6ae-743e-4ce2-af99-e1850a1f88e6"} {"msg":"[2]R generation[0..12]: [72, 70, 75, 74, 74, 74, 72, 72, 74, 73, 74, 69]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:08.159974062Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1440,"":"downstairs","":"downstairs","session_id":"a000f6ae-743e-4ce2-af99-e1850a1f88e6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:08.15998501Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1440,"":"downstairs","":"downstairs","session_id":"a000f6ae-743e-4ce2-af99-e1850a1f88e6"} {"msg":"Max found gen is 76","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:08.159994769Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1440,"":"downstairs","":"downstairs","session_id":"a000f6ae-743e-4ce2-af99-e1850a1f88e6"} {"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:08.160006967Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1440,"":"downstairs","":"downstairs","session_id":"a000f6ae-743e-4ce2-af99-e1850a1f88e6"} {"msg":"Next flush: 129","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:08.160016505Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1440,"":"downstairs","":"downstairs","session_id":"a000f6ae-743e-4ce2-af99-e1850a1f88e6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:08.160026074Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1440,"":"downstairs","":"downstairs","session_id":"a000f6ae-743e-4ce2-af99-e1850a1f88e6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:08.160035802Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1440,"session_id":"a000f6ae-743e-4ce2-af99-e1850a1f88e6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:08.160044611Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1440,"client":"0","":"downstairs","session_id":"a000f6ae-743e-4ce2-af99-e1850a1f88e6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:08.16005453Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1440,"client":"1","":"downstairs","session_id":"a000f6ae-743e-4ce2-af99-e1850a1f88e6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:08.160064328Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1440,"client":"2","":"downstairs","session_id":"a000f6ae-743e-4ce2-af99-e1850a1f88e6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:08.160074317Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1440,"session_id":"a000f6ae-743e-4ce2-af99-e1850a1f88e6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:08.160083545Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1440,"session_id":"a000f6ae-743e-4ce2-af99-e1850a1f88e6"} {"msg":"614384c0-c658-482d-a988-b6eef262afaf is now active with session: a000f6ae-743e-4ce2-af99-e1850a1f88e6","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:08.160093434Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1440,"session_id":"a000f6ae-743e-4ce2-af99-e1850a1f88e6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:08.160102133Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1440,"session_id":"a000f6ae-743e-4ce2-af99-e1850a1f88e6"} {"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:08.160111181Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1440} {"msg":"Activated sub_volume 614384c0-c658-482d-a988-b6eef262afaf","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:08.16011955Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1440} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:08.160127799Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1440,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:08.160450931Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1440,"task":"crutest"} 01/30 Write block 17 len 8192 data: 9 9 02/30 Read block 168 len 24576 03/30 Read block 299 len 4096 04/30 Read block 327 len 20480 05/30 Write block 453 len 20480 data: 9 8 5 7 8 06/30 Write block 322 len 20480 data: 8 9 9 10 10 07/30 Read block 154 len 16384 08/30 Write block 42 len 20480 data: 11 11 11 10 10 09/30 Flush 10/30 Write block 144 len 8192 data: 9 9 11/30 Read block 320 len 20480 12/30 Write block 424 len 28672 data: 6 7 5 8 7 7 6 13/30 Read block 188 len 36864 14/30 Read block 364 len 4096 15/30 Read block 455 len 4096 16/30 Read block 269 len 28672 17/30 Write block 103 len 40960 data: 9 7 8 5 6 5 4 5 8 7 18/30 Read block 360 len 32768 19/30 Flush 20/30 Write block 268 len 40960 data: 7 7 10 12 10 8 9 10 8 9 21/30 Write block 56 len 36864 data: 7 5 5 3 4 3 5 4 4 22/30 Flush 23/30 Write block 536 len 4096 data: 7 24/30 Read block 177 len 40960 25/30 Write block 594 len 24576 data: 10 9 7 4 2 2 26/30 Write block 208 len 32768 data: 10 9 7 8 8 7 7 6 27/30 Write block 113 len 4096 data: 7 28/30 Read block 216 len 12288 29/30 Write block 521 len 4096 data: 8 30/30 Read block 155 len 40960 ---------------------------------------------------------------- Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Read 6 Done Done Done false 1002 Acked Read 1 Sent Done Done false 1003 Acked Read 5 Sent Done Done false 1004 Acked Write 5 Sent Done Done false 1005 Acked Write 5 Sent Done Done false 1006 Acked Read 4 Sent Done Done false 1007 Acked Write 5 Sent Done Done false 1008 Acked Flush 0 Sent Done Done false 1009 Acked Write 2 Sent Done Done false 1010 Acked Read 5 Sent Done Done false 1011 Acked Write 7 Sent Done Done false 1012 Acked Read 9 Sent Done Done false 1013 Acked Read 1 Sent Done Done false 1014 Acked Read 1 Sent Done Done false 1015 Acked Read 7 Sent Done Done false 1016 Acked Write 10 Sent Done Done false 1017 Acked Read 8 Sent Done Done false 1018 Acked Flush 0 Sent Done Done false 1019 Acked Write 10 Sent Done Done false 1020 Acked Write 9 Sent Done Done false 1021 Acked Flush 0 Sent Done Done false 1022 Acked Write 1 Sent Done Done false 1023 Acked Read 10 Sent Done Done false 1024 Acked Write 6 Sent Done Done false 1025 Acked Write 8 Sent Done Done false 1026 Acked Write 1 Sent Done Done false 1027 Acked Read 3 Sent Done Done false 1028 Acked Write 1 Sent Done Done false 1029 Acked Read 10 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 0 0 28 Done 2 30 30 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:56:08.169379254Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1440} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:08.169566226Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1440,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 72 76 76 125 129 129 T F F <--- 1 020-039 70 70 70 123 123 123 F F F 2 040-059 75 76 76 128 131 131 F F F <--- 3 060-079 74 76 76 127 131 131 F F F <--- 4 080-099 74 74 74 127 127 127 F F F 5 100-119 74 76 76 127 130 130 F T T <--- 6 120-139 72 72 72 125 125 125 F F F 7 140-159 72 76 76 125 130 130 F F F <--- 8 160-179 74 74 74 127 127 127 F F F 9 180-199 73 73 73 126 126 126 F F F 10 200-219 74 74 74 127 127 127 F T T <--- 11 220-239 69 69 69 121 121 121 F F F 12 240-259 74 74 74 127 127 127 F F F 13 260-279 74 76 76 127 131 131 F F F <--- 14 280-299 74 74 74 127 127 127 F F F 15 300-319 73 73 73 126 126 126 F F F 16 320-339 70 76 76 122 129 129 F F F <--- 17 340-359 70 70 70 122 122 122 F F F 18 360-379 70 70 70 122 122 122 F F F 19 380-399 74 74 74 127 127 127 F F F 20 400-419 74 74 74 127 127 127 F F F 21 420-439 70 76 76 123 130 130 F F F <--- 22 440-459 74 76 76 127 129 129 F F F <--- 23 460-479 73 73 73 126 126 126 F F F 24 480-499 74 74 74 127 127 127 F F F 25 500-519 73 73 73 126 126 126 F F F 26 520-539 73 73 73 126 126 126 F T T <--- 27 540-559 70 70 70 122 122 122 F F F 28 560-579 60 60 60 111 111 111 F F F 29 580-599 71 71 71 124 124 124 F T T <--- Max gen: 76, Max flush: 131 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 38 Verifying data now No change after verify Loop: 38 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 76 76 76 129 129 129 F F F 1 020-039 70 70 70 123 123 123 F F F 2 040-059 76 76 76 131 131 131 F F F 3 060-079 76 76 76 131 131 131 F F F 4 080-099 74 74 74 127 127 127 F F F 5 100-119 77 77 77 132 132 132 F F F 6 120-139 72 72 72 125 125 125 F F F 7 140-159 76 76 76 130 130 130 F F F 8 160-179 74 74 74 127 127 127 F F F 9 180-199 73 73 73 126 126 126 F F F 10 200-219 77 77 77 132 132 132 F F F 11 220-239 69 69 69 121 121 121 F F F 12 240-259 74 74 74 127 127 127 F F F 13 260-279 76 76 76 131 131 131 F F F 14 280-299 74 74 74 127 127 127 F F F 15 300-319 73 73 73 126 126 126 F F F 16 320-339 76 76 76 129 129 129 F F F 17 340-359 70 70 70 122 122 122 F F F 18 360-379 70 70 70 122 122 122 F F F 19 380-399 74 74 74 127 127 127 F F F 20 400-419 74 74 74 127 127 127 F F F 21 420-439 76 76 76 130 130 130 F F F 22 440-459 76 76 76 129 129 129 F F F 23 460-479 73 73 73 126 126 126 F F F 24 480-499 74 74 74 127 127 127 F F F 25 500-519 73 73 73 126 126 126 F F F 26 520-539 77 77 77 132 132 132 F F F 27 540-559 70 70 70 122 122 122 F F F 28 560-579 60 60 60 111 111 111 F F F 29 580-599 77 77 77 132 132 132 F F F Max gen: 77, Max flush: 132 Begin loop 39 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:12.420503392Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1448,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:56:12.427659546Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1448,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:12.427716698Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1448,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:12.434639316Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1448,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:12.434910246Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1448} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:12.434935113Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1448} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:12.434943521Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1448} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:12.43495497Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1448} {"msg":"Crucible 20decd6f-819b-459b-9abe-2aeea52a8d4f has session id: 55479a95-fb6f-4345-82c8-f53f5e41518f","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:12.435322005Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1448,"session_id":"55479a95-fb6f-4345-82c8-f53f5e41518f"} {"msg":"Upstairs opts: Upstairs UUID: 20decd6f-819b-459b-9abe-2aeea52a8d4f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:12.435368319Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1448,"session_id":"55479a95-fb6f-4345-82c8-f53f5e41518f"} {"msg":"Crucible stats registered with UUID: 20decd6f-819b-459b-9abe-2aeea52a8d4f","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:12.435379037Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1448,"session_id":"55479a95-fb6f-4345-82c8-f53f5e41518f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:56:12.435388016Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1448,"session_id":"55479a95-fb6f-4345-82c8-f53f5e41518f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:12.435527805Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1448,"":"io task","client":"2","":"downstairs","session_id":"55479a95-fb6f-4345-82c8-f53f5e41518f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:12.435543673Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1448,"":"io task","client":"0","":"downstairs","session_id":"55479a95-fb6f-4345-82c8-f53f5e41518f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:12.435553761Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1448,"":"io task","client":"1","":"downstairs","session_id":"55479a95-fb6f-4345-82c8-f53f5e41518f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:12.435694111Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1448,"task":"crutest"} {"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:12.435708858Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1448} {"msg":"20decd6f-819b-459b-9abe-2aeea52a8d4f active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:12.435742533Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1448,"session_id":"55479a95-fb6f-4345-82c8-f53f5e41518f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:12.435751992Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1448,"client":"0","":"downstairs","session_id":"55479a95-fb6f-4345-82c8-f53f5e41518f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:12.435761681Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1448,"client":"1","":"downstairs","session_id":"55479a95-fb6f-4345-82c8-f53f5e41518f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:12.435771409Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1448,"client":"2","":"downstairs","session_id":"55479a95-fb6f-4345-82c8-f53f5e41518f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:12.435781128Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1448,"":"io task","client":"2","":"downstairs","session_id":"55479a95-fb6f-4345-82c8-f53f5e41518f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:12.435791196Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1448,"":"io task","client":"0","":"downstairs","session_id":"55479a95-fb6f-4345-82c8-f53f5e41518f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:12.435819222Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1448,"":"io task","client":"1","":"downstairs","session_id":"55479a95-fb6f-4345-82c8-f53f5e41518f"} {"msg":"ds_connection connected from Ok(127.0.0.1:60195)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:12.435843219Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1448,"":"io task","client":"0","":"downstairs","session_id":"55479a95-fb6f-4345-82c8-f53f5e41518f"} {"msg":"ds_connection connected from Ok(127.0.0.1:43372)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:12.435854737Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1448,"":"io task","client":"2","":"downstairs","session_id":"55479a95-fb6f-4345-82c8-f53f5e41518f"} {"msg":"ds_connection connected from Ok(127.0.0.1:39635)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:12.435934525Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1448,"":"io task","client":"1","":"downstairs","session_id":"55479a95-fb6f-4345-82c8-f53f5e41518f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:12.436583439Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1448,"client":"1","":"downstairs","session_id":"55479a95-fb6f-4345-82c8-f53f5e41518f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:12.436601657Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1448,"client":"2","":"downstairs","session_id":"55479a95-fb6f-4345-82c8-f53f5e41518f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:12.436612705Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1448,"client":"0","":"downstairs","session_id":"55479a95-fb6f-4345-82c8-f53f5e41518f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:12.436898973Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1448,"session_id":"55479a95-fb6f-4345-82c8-f53f5e41518f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:12.436908591Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1448,"session_id":"55479a95-fb6f-4345-82c8-f53f5e41518f"} {"msg":"[0]R flush_numbers[0..12]: [129, 123, 131, 131, 127, 132, 125, 130, 127, 126, 132, 121]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:12.436927629Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1448,"":"downstairs","":"downstairs","session_id":"55479a95-fb6f-4345-82c8-f53f5e41518f"} {"msg":"[0]R generation[0..12]: [76, 70, 76, 76, 74, 77, 72, 76, 74, 73, 77, 69]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:12.436938237Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1448,"":"downstairs","":"downstairs","session_id":"55479a95-fb6f-4345-82c8-f53f5e41518f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:12.436949385Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1448,"":"downstairs","":"downstairs","session_id":"55479a95-fb6f-4345-82c8-f53f5e41518f"} {"msg":"[1]R flush_numbers[0..12]: [129, 123, 131, 131, 127, 132, 125, 130, 127, 126, 132, 121]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:12.436960144Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1448,"":"downstairs","":"downstairs","session_id":"55479a95-fb6f-4345-82c8-f53f5e41518f"} {"msg":"[1]R generation[0..12]: [76, 70, 76, 76, 74, 77, 72, 76, 74, 73, 77, 69]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:12.436970532Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1448,"":"downstairs","":"downstairs","session_id":"55479a95-fb6f-4345-82c8-f53f5e41518f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:12.436981491Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1448,"":"downstairs","":"downstairs","session_id":"55479a95-fb6f-4345-82c8-f53f5e41518f"} {"msg":"[2]R flush_numbers[0..12]: [129, 123, 131, 131, 127, 132, 125, 130, 127, 126, 132, 121]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:12.436992039Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1448,"":"downstairs","":"downstairs","session_id":"55479a95-fb6f-4345-82c8-f53f5e41518f"} {"msg":"[2]R generation[0..12]: [76, 70, 76, 76, 74, 77, 72, 76, 74, 73, 77, 69]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:12.437002478Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1448,"":"downstairs","":"downstairs","session_id":"55479a95-fb6f-4345-82c8-f53f5e41518f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:12.437013456Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1448,"":"downstairs","":"downstairs","session_id":"55479a95-fb6f-4345-82c8-f53f5e41518f"} {"msg":"Max found gen is 78","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:12.437022955Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1448,"":"downstairs","":"downstairs","session_id":"55479a95-fb6f-4345-82c8-f53f5e41518f"} {"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:12.437032823Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1448,"":"downstairs","":"downstairs","session_id":"55479a95-fb6f-4345-82c8-f53f5e41518f"} {"msg":"Next flush: 133","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:12.437042282Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1448,"":"downstairs","":"downstairs","session_id":"55479a95-fb6f-4345-82c8-f53f5e41518f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:12.43705182Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1448,"":"downstairs","":"downstairs","session_id":"55479a95-fb6f-4345-82c8-f53f5e41518f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:12.437061959Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1448,"session_id":"55479a95-fb6f-4345-82c8-f53f5e41518f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:12.437072237Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1448,"client":"0","":"downstairs","session_id":"55479a95-fb6f-4345-82c8-f53f5e41518f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:12.437081916Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1448,"client":"1","":"downstairs","session_id":"55479a95-fb6f-4345-82c8-f53f5e41518f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:12.437091534Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1448,"client":"2","":"downstairs","session_id":"55479a95-fb6f-4345-82c8-f53f5e41518f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:12.437101493Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1448,"session_id":"55479a95-fb6f-4345-82c8-f53f5e41518f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:12.437110612Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1448,"session_id":"55479a95-fb6f-4345-82c8-f53f5e41518f"} {"msg":"20decd6f-819b-459b-9abe-2aeea52a8d4f is now active with session: 55479a95-fb6f-4345-82c8-f53f5e41518f","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:12.43712055Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1448,"session_id":"55479a95-fb6f-4345-82c8-f53f5e41518f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:12.437129439Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1448,"session_id":"55479a95-fb6f-4345-82c8-f53f5e41518f"} {"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:12.437138567Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1448} {"msg":"Activated sub_volume 20decd6f-819b-459b-9abe-2aeea52a8d4f","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:12.437146866Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1448} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:12.437155025Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1448,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:12.437445292Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1448,"task":"crutest"} 01/30 Write block 285 len 24576 data: 7 8 5 5 7 8 02/30 Write block 26 len 4096 data: 11 03/30 Read block 397 len 36864 04/30 Write block 478 len 24576 data: 8 9 9 8 8 6 05/30 Write block 121 len 4096 data: 7 06/30 Read block 130 len 40960 07/30 Read block 107 len 16384 08/30 Read block 300 len 12288 09/30 Write block 353 len 20480 data: 5 8 8 9 9 10/30 Write block 306 len 32768 data: 6 4 5 7 8 11 10 10 11/30 Read block 44 len 24576 12/30 Read block 458 len 36864 13/30 Write block 300 len 16384 data: 5 6 5 5 14/30 Write block 449 len 20480 data: 10 10 11 10 10 15/30 Write block 362 len 36864 data: 4 8 9 11 10 8 7 6 8 16/30 Write block 185 len 40960 data: 4 4 6 5 7 7 12 13 11 11 17/30 Write block 43 len 32768 data: 12 12 11 11 10 8 6 6 18/30 Read block 257 len 32768 19/30 Flush 20/30 Write block 520 len 16384 data: 9 9 7 4 21/30 Read block 516 len 36864 22/30 Read block 145 len 28672 23/30 Read block 189 len 28672 24/30 Read block 13 len 4096 25/30 Write block 336 len 20480 data: 7 8 8 9 7 26/30 Write block 358 len 4096 data: 8 27/30 Write block 466 len 20480 data: 11 10 11 11 9 28/30 Write block 26 len 24576 data: 12 9 9 9 7 8 29/30 Write block 164 len 36864 data: 5 6 6 6 4 4 3 3 4 30/30 Read block 105 len 28672 ---------------------------------------------------------------- Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Sent Done Done false 1001 Acked Write 1 Sent Done Done false 1002 Acked Read 9 Sent Done Done false 1003 Acked Write 6 Sent Done Done false 1004 Acked Write 1 Sent Done Done false 1005 Acked Read 10 Sent Done Done false 1006 Acked Read 4 Sent Done Done false 1007 Acked Read 3 Sent Done Done false 1008 Acked Write 5 Sent Done Done false 1009 Acked Write 8 Sent Done Done false 1010 Acked Read 6 Sent Done Done false 1011 Acked Read 9 Sent Done Done false 1012 Acked Write 4 Sent Done Done false 1013 Acked Write 5 Sent Done Done false 1014 Acked Write 9 Sent Done Done false 1015 Acked Write 10 Sent Done Done false 1016 Acked Write 8 Sent Done Done false 1017 Acked Read 8 Sent Done Done false 1018 Acked Flush 0 Sent Done Done false 1019 Acked Write 4 Sent Done Done false 1020 Acked Read 9 Sent Done Done false 1021 Acked Read 7 Sent Done Done false 1022 Acked Read 7 Sent Done Done false 1023 Acked Read 1 Sent Done Done false 1024 Acked Write 5 Sent Done Done false 1025 Acked Write 1 Sent Done Done false 1026 Acked Write 5 Sent Done Done false 1027 Acked Write 6 Sent Done Done false 1028 Acked Write 9 Sent Done Done false 1029 Acked Read 7 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 0 1 31 Done 0 30 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:56:12.445742758Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1448} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:12.445920592Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1448,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 76 76 76 129 129 129 F F F 1 020-039 70 78 78 123 133 133 F T T <--- 2 040-059 76 78 78 131 133 133 F F F <--- 3 060-079 76 76 76 131 131 131 F F F 4 080-099 74 74 74 127 127 127 F F F 5 100-119 77 77 77 132 132 132 F F F 6 120-139 72 78 78 125 133 133 F F F <--- 7 140-159 76 76 76 130 130 130 F F F 8 160-179 74 74 74 127 127 127 F T T <--- 9 180-199 73 78 78 126 133 133 F F F <--- 10 200-219 77 77 77 132 132 132 F F F 11 220-239 69 69 69 121 121 121 F F F 12 240-259 74 74 74 127 127 127 F F F 13 260-279 76 76 76 131 131 131 F F F 14 280-299 74 78 78 127 133 133 F F F <--- 15 300-319 73 78 78 126 133 133 F F F <--- 16 320-339 76 76 76 129 129 129 F T T <--- 17 340-359 70 78 78 122 133 133 F T T <--- 18 360-379 70 78 78 122 133 133 F F F <--- 19 380-399 74 74 74 127 127 127 F F F 20 400-419 74 74 74 127 127 127 F F F 21 420-439 76 76 76 130 130 130 F F F 22 440-459 76 78 78 129 133 133 F F F <--- 23 460-479 73 78 78 126 133 133 F T T <--- 24 480-499 74 78 78 127 133 133 F F F <--- 25 500-519 73 73 73 126 126 126 F F F 26 520-539 77 77 77 132 132 132 F T T <--- 27 540-559 70 70 70 122 122 122 F F F 28 560-579 60 60 60 111 111 111 F F F 29 580-599 77 77 77 132 132 132 F F F Max gen: 78, Max flush: 133 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 39 Verifying data now No change after verify Loop: 39 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 76 76 76 129 129 129 F F F 1 020-039 79 79 79 134 134 134 F F F 2 040-059 78 78 78 133 133 133 F F F 3 060-079 76 76 76 131 131 131 F F F 4 080-099 74 74 74 127 127 127 F F F 5 100-119 77 77 77 132 132 132 F F F 6 120-139 78 78 78 133 133 133 F F F 7 140-159 76 76 76 130 130 130 F F F 8 160-179 79 79 79 134 134 134 F F F 9 180-199 78 78 78 133 133 133 F F F 10 200-219 77 77 77 132 132 132 F F F 11 220-239 69 69 69 121 121 121 F F F 12 240-259 74 74 74 127 127 127 F F F 13 260-279 76 76 76 131 131 131 F F F 14 280-299 78 78 78 133 133 133 F F F 15 300-319 78 78 78 133 133 133 F F F 16 320-339 79 79 79 134 134 134 F F F 17 340-359 79 79 79 134 134 134 F F F 18 360-379 78 78 78 133 133 133 F F F 19 380-399 74 74 74 127 127 127 F F F 20 400-419 74 74 74 127 127 127 F F F 21 420-439 76 76 76 130 130 130 F F F 22 440-459 78 78 78 133 133 133 F F F 23 460-479 79 79 79 134 134 134 F F F 24 480-499 78 78 78 133 133 133 F F F 25 500-519 73 73 73 126 126 126 F F F 26 520-539 79 79 79 134 134 134 F F F 27 540-559 70 70 70 122 122 122 F F F 28 560-579 60 60 60 111 111 111 F F F 29 580-599 77 77 77 132 132 132 F F F Max gen: 79, Max flush: 134 Begin loop 40 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:16.704077642Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1456,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:16.712306958Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1456,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:16.71263017Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1456} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:16.712663965Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1456} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:16.712675863Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1456} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:16.712687542Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1456} {"msg":"Crucible eb76fd65-6196-44db-8f89-eab77fd4a0fa has session id: 85a57e47-7ce9-4619-9b64-71175a5e428c","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:16.713032851Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1456,"session_id":"85a57e47-7ce9-4619-9b64-71175a5e428c"} {"msg":"Upstairs opts: Upstairs UUID: eb76fd65-6196-44db-8f89-eab77fd4a0fa, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:16.713055497Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1456,"session_id":"85a57e47-7ce9-4619-9b64-71175a5e428c"} {"msg":"Crucible stats registered with UUID: eb76fd65-6196-44db-8f89-eab77fd4a0fa","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:16.713065626Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1456,"session_id":"85a57e47-7ce9-4619-9b64-71175a5e428c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:56:16.713074895Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1456,"session_id":"85a57e47-7ce9-4619-9b64-71175a5e428c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:16.713223933Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1456,"":"io task","client":"1","":"downstairs","session_id":"85a57e47-7ce9-4619-9b64-71175a5e428c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:16.713254368Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1456,"":"io task","client":"0","":"downstairs","session_id":"85a57e47-7ce9-4619-9b64-71175a5e428c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:16.713265406Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1456,"":"io task","client":"2","":"downstairs","session_id":"85a57e47-7ce9-4619-9b64-71175a5e428c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:16.713402086Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1456,"task":"crutest"} {"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:16.713421133Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1456} {"msg":"eb76fd65-6196-44db-8f89-eab77fd4a0fa active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:16.713451749Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1456,"session_id":"85a57e47-7ce9-4619-9b64-71175a5e428c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:16.713461178Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1456,"client":"0","":"downstairs","session_id":"85a57e47-7ce9-4619-9b64-71175a5e428c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:16.713470736Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1456,"client":"1","":"downstairs","session_id":"85a57e47-7ce9-4619-9b64-71175a5e428c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:16.713480535Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1456,"client":"2","":"downstairs","session_id":"85a57e47-7ce9-4619-9b64-71175a5e428c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:16.713490183Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1456,"":"io task","client":"2","":"downstairs","session_id":"85a57e47-7ce9-4619-9b64-71175a5e428c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:16.713502451Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1456,"":"io task","client":"0","":"downstairs","session_id":"85a57e47-7ce9-4619-9b64-71175a5e428c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:16.71351265Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1456,"":"io task","client":"1","":"downstairs","session_id":"85a57e47-7ce9-4619-9b64-71175a5e428c"} {"msg":"ds_connection connected from Ok(127.0.0.1:34305)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:16.713567552Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1456,"":"io task","client":"1","":"downstairs","session_id":"85a57e47-7ce9-4619-9b64-71175a5e428c"} {"msg":"ds_connection connected from Ok(127.0.0.1:53903)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:16.713588599Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1456,"":"io task","client":"0","":"downstairs","session_id":"85a57e47-7ce9-4619-9b64-71175a5e428c"} {"msg":"ds_connection connected from Ok(127.0.0.1:40906)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:16.713683585Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1456,"":"io task","client":"2","":"downstairs","session_id":"85a57e47-7ce9-4619-9b64-71175a5e428c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:16.714307193Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1456,"client":"0","":"downstairs","session_id":"85a57e47-7ce9-4619-9b64-71175a5e428c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:16.714333559Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1456,"client":"1","":"downstairs","session_id":"85a57e47-7ce9-4619-9b64-71175a5e428c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:16.714426015Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1456,"client":"2","":"downstairs","session_id":"85a57e47-7ce9-4619-9b64-71175a5e428c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:16.714612178Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1456,"session_id":"85a57e47-7ce9-4619-9b64-71175a5e428c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:16.714628355Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1456,"session_id":"85a57e47-7ce9-4619-9b64-71175a5e428c"} {"msg":"[0]R flush_numbers[0..12]: [129, 134, 133, 131, 127, 132, 133, 130, 134, 133, 132, 121]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:16.714691106Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1456,"":"downstairs","":"downstairs","session_id":"85a57e47-7ce9-4619-9b64-71175a5e428c"} {"msg":"[0]R generation[0..12]: [76, 79, 78, 76, 74, 77, 78, 76, 79, 78, 77, 69]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:16.714702314Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1456,"":"downstairs","":"downstairs","session_id":"85a57e47-7ce9-4619-9b64-71175a5e428c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:16.714713423Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1456,"":"downstairs","":"downstairs","session_id":"85a57e47-7ce9-4619-9b64-71175a5e428c"} {"msg":"[1]R flush_numbers[0..12]: [129, 134, 133, 131, 127, 132, 133, 130, 134, 133, 132, 121]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:16.714724111Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1456,"":"downstairs","":"downstairs","session_id":"85a57e47-7ce9-4619-9b64-71175a5e428c"} {"msg":"[1]R generation[0..12]: [76, 79, 78, 76, 74, 77, 78, 76, 79, 78, 77, 69]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:16.714736829Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1456,"":"downstairs","":"downstairs","session_id":"85a57e47-7ce9-4619-9b64-71175a5e428c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:16.714747788Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1456,"":"downstairs","":"downstairs","session_id":"85a57e47-7ce9-4619-9b64-71175a5e428c"} {"msg":"[2]R flush_numbers[0..12]: [129, 134, 133, 131, 127, 132, 133, 130, 134, 133, 132, 121]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:16.714758506Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1456,"":"downstairs","":"downstairs","session_id":"85a57e47-7ce9-4619-9b64-71175a5e428c"} {"msg":"[2]R generation[0..12]: [76, 79, 78, 76, 74, 77, 78, 76, 79, 78, 77, 69]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:16.714768885Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1456,"":"downstairs","":"downstairs","session_id":"85a57e47-7ce9-4619-9b64-71175a5e428c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:16.714779953Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1456,"":"downstairs","":"downstairs","session_id":"85a57e47-7ce9-4619-9b64-71175a5e428c"} {"msg":"Max found gen is 80","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:16.714789332Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1456,"":"downstairs","":"downstairs","session_id":"85a57e47-7ce9-4619-9b64-71175a5e428c"} {"msg":"Generation requested: 80 >= found:80","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:16.71479911Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1456,"":"downstairs","":"downstairs","session_id":"85a57e47-7ce9-4619-9b64-71175a5e428c"} {"msg":"Next flush: 135","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:16.714808799Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1456,"":"downstairs","":"downstairs","session_id":"85a57e47-7ce9-4619-9b64-71175a5e428c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:16.714818417Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1456,"":"downstairs","":"downstairs","session_id":"85a57e47-7ce9-4619-9b64-71175a5e428c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:16.714828216Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1456,"session_id":"85a57e47-7ce9-4619-9b64-71175a5e428c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:16.714837155Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1456,"client":"0","":"downstairs","session_id":"85a57e47-7ce9-4619-9b64-71175a5e428c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:16.714846933Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1456,"client":"1","":"downstairs","session_id":"85a57e47-7ce9-4619-9b64-71175a5e428c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:16.714856732Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1456,"client":"2","":"downstairs","session_id":"85a57e47-7ce9-4619-9b64-71175a5e428c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:16.71486668Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1456,"session_id":"85a57e47-7ce9-4619-9b64-71175a5e428c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:16.714875839Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1456,"session_id":"85a57e47-7ce9-4619-9b64-71175a5e428c"} {"msg":"eb76fd65-6196-44db-8f89-eab77fd4a0fa is now active with session: 85a57e47-7ce9-4619-9b64-71175a5e428c","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:16.714885757Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1456,"session_id":"85a57e47-7ce9-4619-9b64-71175a5e428c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:16.714896226Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1456,"session_id":"85a57e47-7ce9-4619-9b64-71175a5e428c"} {"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:16.714905235Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1456} {"msg":"Activated sub_volume eb76fd65-6196-44db-8f89-eab77fd4a0fa","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:16.714913473Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1456} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:16.714921682Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1456,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:16.715227077Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1456,"task":"crutest"} 01/30 Write block 313 len 20480 data: 11 8 9 9 7 02/30 Read block 2 len 32768 03/30 Write block 456 len 8192 data: 8 9 04/30 Read block 70 len 32768 05/30 Write block 88 len 40960 data: 9 8 8 7 7 7 5 6 6 5 06/30 Read block 167 len 36864 07/30 Write block 570 len 20480 data: 4 4 4 5 6 08/30 Write block 352 len 8192 data: 5 6 09/30 Read block 524 len 4096 10/30 Write block 477 len 8192 data: 7 9 11/30 Write block 204 len 32768 data: 6 7 8 12 11 10 8 9 12/30 Flush 13/30 Write block 93 len 32768 data: 8 6 7 7 6 7 7 11 14/30 Read block 272 len 28672 15/30 Read block 254 len 28672 16/30 Read block 461 len 32768 17/30 Write block 357 len 8192 data: 10 9 18/30 Flush 19/30 Write block 441 len 4096 data: 7 20/30 Read block 205 len 16384 21/30 Flush 22/30 Write block 341 len 12288 data: 6 4 5 23/30 Read block 18 len 32768 24/30 Write block 193 len 20480 data: 12 12 9 7 9 25/30 Flush 26/30 Write block 541 len 40960 data: 6 6 5 5 6 6 4 4 4 4 27/30 Read block 509 len 4096 28/30 Write block 119 len 24576 data: 7 6 8 6 7 4 29/30 Read block 98 len 24576 30/30 Write block 304 len 36864 data: 5 6 7 5 6 8 9 12 11 ---------------------------------------------------------------- Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Read 8 Done Done Done false 1002 Acked Write 2 Done Done Done false 1003 Acked Read 8 Done Sent Done false 1004 Acked Write 10 Done Sent Done false 1005 Acked Read 9 Done Sent Done false 1006 Acked Write 5 Done Sent Done false 1007 Acked Write 2 Done Sent Done false 1008 Acked Read 1 Done Sent Done false 1009 Acked Write 2 Done Sent Done false 1010 Acked Write 8 Done Sent Done false 1011 Acked Flush 0 Done Sent Done false 1012 Acked Write 8 Done Sent Done false 1013 Acked Read 7 Done Sent Done false 1014 Acked Read 7 Done Sent Done false 1015 Acked Read 8 Done Sent Done false 1016 Acked Write 2 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Write 1 Done Sent Done false 1019 Acked Read 4 Done Sent Done false 1020 Acked Flush 0 Done Sent Done false 1021 Acked Write 3 Done Sent Done false 1022 Acked Read 8 Done Sent Done false 1023 Acked Write 5 Done Sent Done false 1024 Acked Flush 0 Done Sent Done false 1025 Acked Write 10 Done Sent Done false 1026 Acked Read 1 Done Sent Done false 1027 Acked Write 6 Done Sent Done false 1028 Acked Read 6 Done Sent Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 27 1 29 Done 29 3 29 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) None Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:56:16.724286001Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1456} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:16.72449046Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1456,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 76 76 76 129 129 129 F F F 1 020-039 79 79 79 134 134 134 F F F 2 040-059 78 78 78 133 133 133 F F F 3 060-079 76 76 76 131 131 131 F F F 4 080-099 80 74 80 136 127 136 F F F <--- 5 100-119 80 77 80 136 132 136 T F T <--- 6 120-139 78 78 78 133 133 133 T F T <--- 7 140-159 76 76 76 130 130 130 F F F 8 160-179 79 79 79 134 134 134 F F F 9 180-199 80 78 80 138 133 138 F F F <--- 10 200-219 80 77 80 135 132 135 F F F <--- 11 220-239 69 69 69 121 121 121 F F F 12 240-259 74 74 74 127 127 127 F F F 13 260-279 76 76 76 131 131 131 F F F 14 280-299 78 78 78 133 133 133 F F F 15 300-319 80 78 80 135 133 135 T T T <--- 16 320-339 79 79 79 134 134 134 F F F 17 340-359 80 79 80 138 134 138 F F F <--- 18 360-379 78 78 78 133 133 133 F F F 19 380-399 74 74 74 127 127 127 F F F 20 400-419 74 74 74 127 127 127 F F F 21 420-439 76 76 76 130 130 130 F F F 22 440-459 80 78 80 137 133 137 F T F <--- 23 460-479 80 79 80 135 134 135 F F F <--- 24 480-499 78 78 78 133 133 133 F F F 25 500-519 73 73 73 126 126 126 F F F 26 520-539 79 79 79 134 134 134 F F F 27 540-559 70 70 70 122 122 122 T F T <--- 28 560-579 80 60 80 135 111 135 F F F <--- 29 580-599 77 77 77 132 132 132 F F F Max gen: 80, Max flush: 138 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 40 Verifying data now No change after verify Loop: 40 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 76 76 76 129 129 129 F F F 1 020-039 79 79 79 134 134 134 F F F 2 040-059 78 78 78 133 133 133 F F F 3 060-079 76 76 76 131 131 131 F F F 4 080-099 80 80 80 136 136 136 F F F 5 100-119 81 81 81 139 139 139 F F F 6 120-139 81 81 81 139 139 139 F F F 7 140-159 76 76 76 130 130 130 F F F 8 160-179 79 79 79 134 134 134 F F F 9 180-199 80 80 80 138 138 138 F F F 10 200-219 80 80 80 135 135 135 F F F 11 220-239 69 69 69 121 121 121 F F F 12 240-259 74 74 74 127 127 127 F F F 13 260-279 76 76 76 131 131 131 F F F 14 280-299 78 78 78 133 133 133 F F F 15 300-319 81 81 81 139 139 139 F F F 16 320-339 79 79 79 134 134 134 F F F 17 340-359 80 80 80 138 138 138 F F F 18 360-379 78 78 78 133 133 133 F F F 19 380-399 74 74 74 127 127 127 F F F 20 400-419 74 74 74 127 127 127 F F F 21 420-439 76 76 76 130 130 130 F F F 22 440-459 80 80 80 137 137 137 F F F 23 460-479 80 80 80 135 135 135 F F F 24 480-499 78 78 78 133 133 133 F F F 25 500-519 73 73 73 126 126 126 F F F 26 520-539 79 79 79 134 134 134 F F F 27 540-559 81 81 81 139 139 139 F F F 28 560-579 80 80 80 135 135 135 F F F 29 580-599 77 77 77 132 132 132 F F F Max gen: 81, Max flush: 139 Begin loop 41 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:20.970829723Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1464,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:20.978974832Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1464,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:20.979261449Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1464} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:20.979290935Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1464} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:20.979299564Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1464} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:20.979311022Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1464} {"msg":"Crucible 73386582-a21f-4b81-a72c-5965b42b8820 has session id: f7164a91-efba-4958-8c1a-7f0d2a9b0997","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:20.979614207Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1464,"session_id":"f7164a91-efba-4958-8c1a-7f0d2a9b0997"} {"msg":"Upstairs opts: Upstairs UUID: 73386582-a21f-4b81-a72c-5965b42b8820, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:20.979643293Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1464,"session_id":"f7164a91-efba-4958-8c1a-7f0d2a9b0997"} {"msg":"Crucible stats registered with UUID: 73386582-a21f-4b81-a72c-5965b42b8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:20.979654621Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1464,"session_id":"f7164a91-efba-4958-8c1a-7f0d2a9b0997"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:56:20.97966677Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1464,"session_id":"f7164a91-efba-4958-8c1a-7f0d2a9b0997"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:20.979792371Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1464,"":"io task","client":"2","":"downstairs","session_id":"f7164a91-efba-4958-8c1a-7f0d2a9b0997"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:20.979808959Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1464,"":"io task","client":"0","":"downstairs","session_id":"f7164a91-efba-4958-8c1a-7f0d2a9b0997"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:20.979819127Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1464,"":"io task","client":"1","":"downstairs","session_id":"f7164a91-efba-4958-8c1a-7f0d2a9b0997"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:20.979972784Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1464,"task":"crutest"} {"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:20.979998151Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1464} {"msg":"73386582-a21f-4b81-a72c-5965b42b8820 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:20.980015188Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1464,"session_id":"f7164a91-efba-4958-8c1a-7f0d2a9b0997"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:20.980024187Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1464,"client":"0","":"downstairs","session_id":"f7164a91-efba-4958-8c1a-7f0d2a9b0997"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:20.980034075Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1464,"client":"1","":"downstairs","session_id":"f7164a91-efba-4958-8c1a-7f0d2a9b0997"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:20.980043524Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1464,"client":"2","":"downstairs","session_id":"f7164a91-efba-4958-8c1a-7f0d2a9b0997"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:20.980053303Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1464,"":"io task","client":"2","":"downstairs","session_id":"f7164a91-efba-4958-8c1a-7f0d2a9b0997"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:20.980063201Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1464,"":"io task","client":"0","":"downstairs","session_id":"f7164a91-efba-4958-8c1a-7f0d2a9b0997"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:20.98007307Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1464,"":"io task","client":"1","":"downstairs","session_id":"f7164a91-efba-4958-8c1a-7f0d2a9b0997"} {"msg":"ds_connection connected from Ok(127.0.0.1:58805)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:20.980104275Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1464,"":"io task","client":"2","":"downstairs","session_id":"f7164a91-efba-4958-8c1a-7f0d2a9b0997"} {"msg":"ds_connection connected from Ok(127.0.0.1:39482)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:20.980115423Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1464,"":"io task","client":"1","":"downstairs","session_id":"f7164a91-efba-4958-8c1a-7f0d2a9b0997"} {"msg":"ds_connection connected from Ok(127.0.0.1:39564)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:20.980125722Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1464,"":"io task","client":"0","":"downstairs","session_id":"f7164a91-efba-4958-8c1a-7f0d2a9b0997"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:20.980709416Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1464,"client":"0","":"downstairs","session_id":"f7164a91-efba-4958-8c1a-7f0d2a9b0997"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:20.980724784Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1464,"client":"1","":"downstairs","session_id":"f7164a91-efba-4958-8c1a-7f0d2a9b0997"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:20.98074544Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1464,"client":"2","":"downstairs","session_id":"f7164a91-efba-4958-8c1a-7f0d2a9b0997"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:20.980980946Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1464,"session_id":"f7164a91-efba-4958-8c1a-7f0d2a9b0997"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:20.981009651Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1464,"session_id":"f7164a91-efba-4958-8c1a-7f0d2a9b0997"} {"msg":"[0]R flush_numbers[0..12]: [129, 134, 133, 131, 136, 139, 139, 130, 134, 138, 135, 121]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:20.981051965Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1464,"":"downstairs","":"downstairs","session_id":"f7164a91-efba-4958-8c1a-7f0d2a9b0997"} {"msg":"[0]R generation[0..12]: [76, 79, 78, 76, 80, 81, 81, 76, 79, 80, 80, 69]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:20.981063064Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1464,"":"downstairs","":"downstairs","session_id":"f7164a91-efba-4958-8c1a-7f0d2a9b0997"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:20.981073962Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1464,"":"downstairs","":"downstairs","session_id":"f7164a91-efba-4958-8c1a-7f0d2a9b0997"} {"msg":"[1]R flush_numbers[0..12]: [129, 134, 133, 131, 136, 139, 139, 130, 134, 138, 135, 121]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:20.98108475Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1464,"":"downstairs","":"downstairs","session_id":"f7164a91-efba-4958-8c1a-7f0d2a9b0997"} {"msg":"[1]R generation[0..12]: [76, 79, 78, 76, 80, 81, 81, 76, 79, 80, 80, 69]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:20.981095229Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1464,"":"downstairs","":"downstairs","session_id":"f7164a91-efba-4958-8c1a-7f0d2a9b0997"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:20.981106077Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1464,"":"downstairs","":"downstairs","session_id":"f7164a91-efba-4958-8c1a-7f0d2a9b0997"} {"msg":"[2]R flush_numbers[0..12]: [129, 134, 133, 131, 136, 139, 139, 130, 134, 138, 135, 121]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:20.981116786Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1464,"":"downstairs","":"downstairs","session_id":"f7164a91-efba-4958-8c1a-7f0d2a9b0997"} {"msg":"[2]R generation[0..12]: [76, 79, 78, 76, 80, 81, 81, 76, 79, 80, 80, 69]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:20.981127424Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1464,"":"downstairs","":"downstairs","session_id":"f7164a91-efba-4958-8c1a-7f0d2a9b0997"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:20.981138402Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1464,"":"downstairs","":"downstairs","session_id":"f7164a91-efba-4958-8c1a-7f0d2a9b0997"} {"msg":"Max found gen is 82","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:20.981150041Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1464,"":"downstairs","":"downstairs","session_id":"f7164a91-efba-4958-8c1a-7f0d2a9b0997"} {"msg":"Generation requested: 82 >= found:82","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:20.981159519Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1464,"":"downstairs","":"downstairs","session_id":"f7164a91-efba-4958-8c1a-7f0d2a9b0997"} {"msg":"Next flush: 140","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:20.981168998Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1464,"":"downstairs","":"downstairs","session_id":"f7164a91-efba-4958-8c1a-7f0d2a9b0997"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:20.981178437Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1464,"":"downstairs","":"downstairs","session_id":"f7164a91-efba-4958-8c1a-7f0d2a9b0997"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:20.981188395Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1464,"session_id":"f7164a91-efba-4958-8c1a-7f0d2a9b0997"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:20.981197304Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1464,"client":"0","":"downstairs","session_id":"f7164a91-efba-4958-8c1a-7f0d2a9b0997"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:20.981207312Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1464,"client":"1","":"downstairs","session_id":"f7164a91-efba-4958-8c1a-7f0d2a9b0997"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:20.981217201Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1464,"client":"2","":"downstairs","session_id":"f7164a91-efba-4958-8c1a-7f0d2a9b0997"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:20.981227349Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1464,"session_id":"f7164a91-efba-4958-8c1a-7f0d2a9b0997"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:20.981236588Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1464,"session_id":"f7164a91-efba-4958-8c1a-7f0d2a9b0997"} {"msg":"73386582-a21f-4b81-a72c-5965b42b8820 is now active with session: f7164a91-efba-4958-8c1a-7f0d2a9b0997","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:20.981246507Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1464,"session_id":"f7164a91-efba-4958-8c1a-7f0d2a9b0997"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:20.981255415Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1464,"session_id":"f7164a91-efba-4958-8c1a-7f0d2a9b0997"} {"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:20.981264424Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1464} {"msg":"Activated sub_volume 73386582-a21f-4b81-a72c-5965b42b8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:20.981272433Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1464} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:20.981280492Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1464,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:20.9815591Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1464,"task":"crutest"} 01/30 Write block 348 len 28672 data: 4 3 4 5 6 7 9 02/30 Read block 431 len 12288 03/30 Read block 526 len 36864 04/30 Write block 546 len 36864 data: 7 5 5 5 5 4 6 6 6 05/30 Write block 284 len 4096 data: 8 06/30 Read block 450 len 20480 07/30 Read block 407 len 8192 08/30 Write block 14 len 8192 data: 5 8 09/30 Write block 111 len 8192 data: 9 8 10/30 Read block 240 len 12288 11/30 Read block 204 len 12288 12/30 Read block 112 len 32768 13/30 Write block 79 len 12288 data: 6 5 3 14/30 Write block 495 len 24576 data: 6 6 5 4 6 6 15/30 Write block 221 len 8192 data: 6 6 16/30 Flush 17/30 Write block 534 len 24576 data: 7 7 8 5 6 6 18/30 Read block 466 len 20480 19/30 Read block 381 len 16384 20/30 Write block 513 len 8192 data: 9 9 21/30 Read block 351 len 8192 22/30 Read block 511 len 36864 23/30 Write block 67 len 36864 data: 6 5 5 6 6 6 6 6 5 24/30 Read block 94 len 16384 25/30 Flush 26/30 Write block 229 len 40960 data: 4 4 3 4 5 4 4 4 4 4 27/30 Read block 351 len 20480 28/30 Read block 516 len 16384 29/30 Write block 574 len 8192 data: 7 6 30/30 Read block 340 len 12288 ---------------------------------------------------------------- Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Sent Done false 1001 Acked Read 3 Done Sent Done false 1002 Acked Read 9 Done Sent Done false 1003 Acked Write 9 Done Sent Done false 1004 Acked Write 1 Done Sent Done false 1005 Acked Read 5 Done Sent Done false 1006 Acked Read 2 Done Sent Done false 1007 Acked Write 2 Done Sent Done false 1008 Acked Write 2 Done Sent Done false 1009 Acked Read 3 Done Sent Done false 1010 Acked Read 3 Done Sent Done false 1011 Acked Read 8 Done Sent Done false 1012 Acked Write 3 Done Sent Done false 1013 Acked Write 6 Done Sent Done false 1014 Acked Write 2 Done Sent Done false 1015 Acked Flush 0 Done Sent Done false 1016 Acked Write 6 Done Sent Done false 1017 Acked Read 5 Done Sent Done false 1018 Acked Read 4 Done Sent Done false 1019 Acked Write 2 Done Sent Done false 1020 Acked Read 2 Done Sent Done false 1021 Acked Read 9 Done Sent Done false 1022 Acked Write 9 Done Sent Done false 1023 Acked Read 4 Done Sent Done false 1024 Acked Flush 0 Done Sent Done false 1025 Acked Write 10 Done Sent Done false 1026 Acked Read 5 Done Sent Done false 1027 Acked Read 4 Done Sent Done false 1028 Acked Write 2 Done Sent Done false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 30 1 31 Done 30 0 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) None Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:56:20.989891141Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1464} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:20.990073264Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1464,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 82 76 82 140 129 140 F F F <--- 1 020-039 79 79 79 134 134 134 F F F 2 040-059 78 78 78 133 133 133 F F F 3 060-079 82 76 82 141 131 141 F F F <--- 4 080-099 82 80 82 140 136 140 F F F <--- 5 100-119 82 81 82 140 139 140 F F F <--- 6 120-139 81 81 81 139 139 139 F F F 7 140-159 76 76 76 130 130 130 F F F 8 160-179 79 79 79 134 134 134 F F F 9 180-199 80 80 80 138 138 138 F F F 10 200-219 80 80 80 135 135 135 F F F 11 220-239 82 69 82 140 121 140 T F T <--- 12 240-259 74 74 74 127 127 127 F F F 13 260-279 76 76 76 131 131 131 F F F 14 280-299 82 78 82 140 133 140 F F F <--- 15 300-319 81 81 81 139 139 139 F F F 16 320-339 79 79 79 134 134 134 F F F 17 340-359 82 80 82 140 138 140 F F F <--- 18 360-379 78 78 78 133 133 133 F F F 19 380-399 74 74 74 127 127 127 F F F 20 400-419 74 74 74 127 127 127 F F F 21 420-439 76 76 76 130 130 130 F F F 22 440-459 80 80 80 137 137 137 F F F 23 460-479 80 80 80 135 135 135 F F F 24 480-499 82 78 82 140 133 140 F F F <--- 25 500-519 82 73 82 141 126 141 F F F <--- 26 520-539 82 79 82 141 134 141 F F F <--- 27 540-559 82 81 82 140 139 140 F F F <--- 28 560-579 80 80 80 135 135 135 T F T <--- 29 580-599 77 77 77 132 132 132 F F F Max gen: 82, Max flush: 141 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 41 Verifying data now No change after verify Loop: 41 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 82 82 82 140 140 140 F F F 1 020-039 79 79 79 134 134 134 F F F 2 040-059 78 78 78 133 133 133 F F F 3 060-079 82 82 82 141 141 141 F F F 4 080-099 82 82 82 140 140 140 F F F 5 100-119 82 82 82 140 140 140 F F F 6 120-139 81 81 81 139 139 139 F F F 7 140-159 76 76 76 130 130 130 F F F 8 160-179 79 79 79 134 134 134 F F F 9 180-199 80 80 80 138 138 138 F F F 10 200-219 80 80 80 135 135 135 F F F 11 220-239 83 83 83 142 142 142 F F F 12 240-259 74 74 74 127 127 127 F F F 13 260-279 76 76 76 131 131 131 F F F 14 280-299 82 82 82 140 140 140 F F F 15 300-319 81 81 81 139 139 139 F F F 16 320-339 79 79 79 134 134 134 F F F 17 340-359 82 82 82 140 140 140 F F F 18 360-379 78 78 78 133 133 133 F F F 19 380-399 74 74 74 127 127 127 F F F 20 400-419 74 74 74 127 127 127 F F F 21 420-439 76 76 76 130 130 130 F F F 22 440-459 80 80 80 137 137 137 F F F 23 460-479 80 80 80 135 135 135 F F F 24 480-499 82 82 82 140 140 140 F F F 25 500-519 82 82 82 141 141 141 F F F 26 520-539 82 82 82 141 141 141 F F F 27 540-559 82 82 82 140 140 140 F F F 28 560-579 83 83 83 142 142 142 F F F 29 580-599 77 77 77 132 132 132 F F F Max gen: 83, Max flush: 142 Begin loop 42 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:25.23240957Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1472,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:56:25.239671418Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1472,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:25.239723801Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1472,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:25.246621883Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1472,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:25.246878395Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1472} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:25.246902071Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1472} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:25.2469104Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1472} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:25.246922158Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1472} {"msg":"Crucible f34efec2-0fb2-4d14-b6be-a735b7eb0f41 has session id: 3f9e6364-ec46-4046-87ff-eb86207c9eae","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:25.247246301Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1472,"session_id":"3f9e6364-ec46-4046-87ff-eb86207c9eae"} {"msg":"Upstairs opts: Upstairs UUID: f34efec2-0fb2-4d14-b6be-a735b7eb0f41, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:25.247296363Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1472,"session_id":"3f9e6364-ec46-4046-87ff-eb86207c9eae"} {"msg":"Crucible stats registered with UUID: f34efec2-0fb2-4d14-b6be-a735b7eb0f41","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:25.247306842Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1472,"session_id":"3f9e6364-ec46-4046-87ff-eb86207c9eae"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:56:25.24731568Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1472,"session_id":"3f9e6364-ec46-4046-87ff-eb86207c9eae"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:25.247448531Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1472,"":"io task","client":"1","":"downstairs","session_id":"3f9e6364-ec46-4046-87ff-eb86207c9eae"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:25.247465208Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1472,"":"io task","client":"0","":"downstairs","session_id":"3f9e6364-ec46-4046-87ff-eb86207c9eae"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:25.247475417Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1472,"":"io task","client":"2","":"downstairs","session_id":"3f9e6364-ec46-4046-87ff-eb86207c9eae"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:25.247626465Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1472,"task":"crutest"} {"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:25.247641852Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1472} {"msg":"f34efec2-0fb2-4d14-b6be-a735b7eb0f41 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:25.247673448Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1472,"session_id":"3f9e6364-ec46-4046-87ff-eb86207c9eae"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:25.247682836Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1472,"client":"0","":"downstairs","session_id":"3f9e6364-ec46-4046-87ff-eb86207c9eae"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:25.247692625Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1472,"client":"1","":"downstairs","session_id":"3f9e6364-ec46-4046-87ff-eb86207c9eae"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:25.247702063Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1472,"client":"2","":"downstairs","session_id":"3f9e6364-ec46-4046-87ff-eb86207c9eae"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:25.247711592Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1472,"":"io task","client":"2","":"downstairs","session_id":"3f9e6364-ec46-4046-87ff-eb86207c9eae"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:25.247721681Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1472,"":"io task","client":"0","":"downstairs","session_id":"3f9e6364-ec46-4046-87ff-eb86207c9eae"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:25.247731649Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1472,"":"io task","client":"1","":"downstairs","session_id":"3f9e6364-ec46-4046-87ff-eb86207c9eae"} {"msg":"ds_connection connected from Ok(127.0.0.1:60868)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:25.247777872Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1472,"":"io task","client":"2","":"downstairs","session_id":"3f9e6364-ec46-4046-87ff-eb86207c9eae"} {"msg":"ds_connection connected from Ok(127.0.0.1:38490)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:25.247788801Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1472,"":"io task","client":"0","":"downstairs","session_id":"3f9e6364-ec46-4046-87ff-eb86207c9eae"} {"msg":"ds_connection connected from Ok(127.0.0.1:33917)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:25.247799189Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1472,"":"io task","client":"1","":"downstairs","session_id":"3f9e6364-ec46-4046-87ff-eb86207c9eae"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:25.248399091Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1472,"client":"2","":"downstairs","session_id":"3f9e6364-ec46-4046-87ff-eb86207c9eae"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:25.248416608Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1472,"client":"1","":"downstairs","session_id":"3f9e6364-ec46-4046-87ff-eb86207c9eae"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:25.248532901Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1472,"client":"0","":"downstairs","session_id":"3f9e6364-ec46-4046-87ff-eb86207c9eae"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:25.248689298Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1472,"session_id":"3f9e6364-ec46-4046-87ff-eb86207c9eae"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:25.248707445Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1472,"session_id":"3f9e6364-ec46-4046-87ff-eb86207c9eae"} {"msg":"[0]R flush_numbers[0..12]: [140, 134, 133, 141, 140, 140, 139, 130, 134, 138, 135, 142]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:25.248797072Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1472,"":"downstairs","":"downstairs","session_id":"3f9e6364-ec46-4046-87ff-eb86207c9eae"} {"msg":"[0]R generation[0..12]: [82, 79, 78, 82, 82, 82, 81, 76, 79, 80, 80, 83]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:25.24880821Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1472,"":"downstairs","":"downstairs","session_id":"3f9e6364-ec46-4046-87ff-eb86207c9eae"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:25.248819429Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1472,"":"downstairs","":"downstairs","session_id":"3f9e6364-ec46-4046-87ff-eb86207c9eae"} {"msg":"[1]R flush_numbers[0..12]: [140, 134, 133, 141, 140, 140, 139, 130, 134, 138, 135, 142]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:25.248830087Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1472,"":"downstairs","":"downstairs","session_id":"3f9e6364-ec46-4046-87ff-eb86207c9eae"} {"msg":"[1]R generation[0..12]: [82, 79, 78, 82, 82, 82, 81, 76, 79, 80, 80, 83]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:25.248840395Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1472,"":"downstairs","":"downstairs","session_id":"3f9e6364-ec46-4046-87ff-eb86207c9eae"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:25.248851444Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1472,"":"downstairs","":"downstairs","session_id":"3f9e6364-ec46-4046-87ff-eb86207c9eae"} {"msg":"[2]R flush_numbers[0..12]: [140, 134, 133, 141, 140, 140, 139, 130, 134, 138, 135, 142]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:25.248862212Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1472,"":"downstairs","":"downstairs","session_id":"3f9e6364-ec46-4046-87ff-eb86207c9eae"} {"msg":"[2]R generation[0..12]: [82, 79, 78, 82, 82, 82, 81, 76, 79, 80, 80, 83]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:25.248872311Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1472,"":"downstairs","":"downstairs","session_id":"3f9e6364-ec46-4046-87ff-eb86207c9eae"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:25.248883279Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1472,"":"downstairs","":"downstairs","session_id":"3f9e6364-ec46-4046-87ff-eb86207c9eae"} {"msg":"Max found gen is 84","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:25.248892808Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1472,"":"downstairs","":"downstairs","session_id":"3f9e6364-ec46-4046-87ff-eb86207c9eae"} {"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:25.248902456Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1472,"":"downstairs","":"downstairs","session_id":"3f9e6364-ec46-4046-87ff-eb86207c9eae"} {"msg":"Next flush: 143","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:25.248911605Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1472,"":"downstairs","":"downstairs","session_id":"3f9e6364-ec46-4046-87ff-eb86207c9eae"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:25.248921044Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1472,"":"downstairs","":"downstairs","session_id":"3f9e6364-ec46-4046-87ff-eb86207c9eae"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:25.248932452Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1472,"session_id":"3f9e6364-ec46-4046-87ff-eb86207c9eae"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:25.248941071Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1472,"client":"0","":"downstairs","session_id":"3f9e6364-ec46-4046-87ff-eb86207c9eae"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:25.248950839Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1472,"client":"1","":"downstairs","session_id":"3f9e6364-ec46-4046-87ff-eb86207c9eae"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:25.248960538Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1472,"client":"2","":"downstairs","session_id":"3f9e6364-ec46-4046-87ff-eb86207c9eae"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:25.248970296Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1472,"session_id":"3f9e6364-ec46-4046-87ff-eb86207c9eae"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:25.248979325Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1472,"session_id":"3f9e6364-ec46-4046-87ff-eb86207c9eae"} {"msg":"f34efec2-0fb2-4d14-b6be-a735b7eb0f41 is now active with session: 3f9e6364-ec46-4046-87ff-eb86207c9eae","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:25.248989164Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1472,"session_id":"3f9e6364-ec46-4046-87ff-eb86207c9eae"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:25.248997822Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1472,"session_id":"3f9e6364-ec46-4046-87ff-eb86207c9eae"} {"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:25.249006791Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1472} {"msg":"Activated sub_volume f34efec2-0fb2-4d14-b6be-a735b7eb0f41","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:25.24901489Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1472} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:25.249023019Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1472,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:25.249335702Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1472,"task":"crutest"} 01/30 Write block 235 len 40960 data: 5 5 5 5 4 6 6 6 7 9 02/30 Read block 109 len 8192 03/30 Write block 157 len 24576 data: 7 6 6 6 6 6 04/30 Read block 377 len 24576 05/30 Read block 371 len 4096 06/30 Write block 411 len 16384 data: 15 13 14 10 07/30 Read block 514 len 20480 08/30 Read block 287 len 24576 09/30 Read block 259 len 4096 10/30 Write block 363 len 40960 data: 9 10 12 11 9 8 7 9 7 8 11/30 Write block 582 len 12288 data: 9 9 10 12/30 Read block 41 len 32768 13/30 Write block 422 len 40960 data: 6 6 7 8 6 9 8 8 7 5 14/30 Read block 479 len 24576 15/30 Write block 549 len 28672 data: 6 6 5 7 7 7 6 16/30 Read block 571 len 32768 17/30 Read block 435 len 36864 18/30 Write block 19 len 16384 data: 10 12 11 12 19/30 Write block 108 len 32768 data: 6 5 6 10 9 8 9 9 20/30 Read block 570 len 16384 21/30 Read block 272 len 12288 22/30 Flush 23/30 Write block 436 len 36864 data: 8 9 10 8 6 8 6 6 8 24/30 Read block 99 len 20480 25/30 Read block 360 len 8192 26/30 Write block 396 len 20480 data: 5 6 5 5 5 27/30 Read block 538 len 12288 28/30 Write block 299 len 16384 data: 3 6 7 6 29/30 Write block 309 len 32768 data: 9 10 13 12 12 9 10 10 30/30 Read block 442 len 12288 ---------------------------------------------------------------- Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Sent Done Done false 1001 Acked Read 2 Sent Done Done false 1002 Acked Write 6 Sent Done Done false 1003 Acked Read 6 Sent Done Done false 1004 Acked Read 1 Sent Done Done false 1005 Acked Write 4 Sent Done Done false 1006 Acked Read 5 Sent Done Done false 1007 Acked Read 6 Sent Done Done false 1008 Acked Read 1 Sent Done Done false 1009 Acked Write 10 Sent Done Done false 1010 Acked Write 3 Sent Done Done false 1011 Acked Read 8 Sent Done Done false 1012 Acked Write 10 Sent Done Done false 1013 Acked Read 6 Sent Done Done false 1014 Acked Write 7 Sent Done Done false 1015 Acked Read 8 Sent Done Done false 1016 Acked Read 9 Sent Done Done false 1017 Acked Write 4 Sent Done Done false 1018 Acked Write 8 Sent Done Done false 1019 Acked Read 4 Sent Done Done false 1020 Acked Read 3 Sent Done Done false 1021 Acked Flush 0 Sent Done Done false 1022 Acked Write 9 Sent Done Done false 1023 Acked Read 5 Sent Done Done false 1024 Acked Read 2 Sent Done Done false 1025 Acked Write 5 Sent Done Done false 1026 Acked Read 3 Sent Done Done false 1027 Acked Write 4 Sent Done Done false 1028 Acked Write 8 Sent Done Done false 1029 Acked Read 3 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 0 1 31 Done 0 30 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:56:25.25904273Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1472} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:25.259215205Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1472,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 82 84 84 140 143 143 F F F <--- 1 020-039 79 84 84 134 143 143 F F F <--- 2 040-059 78 78 78 133 133 133 F F F 3 060-079 82 82 82 141 141 141 F F F 4 080-099 82 82 82 140 140 140 F F F 5 100-119 82 84 84 140 143 143 F F F <--- 6 120-139 81 81 81 139 139 139 F F F 7 140-159 76 84 84 130 143 143 F F F <--- 8 160-179 79 84 84 134 143 143 F F F <--- 9 180-199 80 80 80 138 138 138 F F F 10 200-219 80 80 80 135 135 135 F F F 11 220-239 83 84 84 142 143 143 F F F <--- 12 240-259 74 84 84 127 143 143 F F F <--- 13 260-279 76 76 76 131 131 131 F F F 14 280-299 82 82 82 140 140 140 F T T <--- 15 300-319 81 81 81 139 139 139 F T T <--- 16 320-339 79 79 79 134 134 134 F F F 17 340-359 82 82 82 140 140 140 F F F 18 360-379 78 84 84 133 143 143 F F F <--- 19 380-399 74 74 74 127 127 127 F T T <--- 20 400-419 74 84 84 127 143 143 F T T <--- 21 420-439 76 84 84 130 143 143 F T T <--- 22 440-459 80 80 80 137 137 137 F T T <--- 23 460-479 80 80 80 135 135 135 F F F 24 480-499 82 82 82 140 140 140 F F F 25 500-519 82 82 82 141 141 141 F F F 26 520-539 82 82 82 141 141 141 F F F 27 540-559 82 84 84 140 143 143 F F F <--- 28 560-579 83 83 83 142 142 142 F F F 29 580-599 77 84 84 132 143 143 F F F <--- Max gen: 84, Max flush: 143 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 42 Verifying data now No change after verify Loop: 42 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 84 84 84 143 143 143 F F F 1 020-039 84 84 84 143 143 143 F F F 2 040-059 78 78 78 133 133 133 F F F 3 060-079 82 82 82 141 141 141 F F F 4 080-099 82 82 82 140 140 140 F F F 5 100-119 84 84 84 143 143 143 F F F 6 120-139 81 81 81 139 139 139 F F F 7 140-159 84 84 84 143 143 143 F F F 8 160-179 84 84 84 143 143 143 F F F 9 180-199 80 80 80 138 138 138 F F F 10 200-219 80 80 80 135 135 135 F F F 11 220-239 84 84 84 143 143 143 F F F 12 240-259 84 84 84 143 143 143 F F F 13 260-279 76 76 76 131 131 131 F F F 14 280-299 85 85 85 144 144 144 F F F 15 300-319 85 85 85 144 144 144 F F F 16 320-339 79 79 79 134 134 134 F F F 17 340-359 82 82 82 140 140 140 F F F 18 360-379 84 84 84 143 143 143 F F F 19 380-399 85 85 85 144 144 144 F F F 20 400-419 85 85 85 144 144 144 F F F 21 420-439 85 85 85 144 144 144 F F F 22 440-459 85 85 85 144 144 144 F F F 23 460-479 80 80 80 135 135 135 F F F 24 480-499 82 82 82 140 140 140 F F F 25 500-519 82 82 82 141 141 141 F F F 26 520-539 82 82 82 141 141 141 F F F 27 540-559 84 84 84 143 143 143 F F F 28 560-579 83 83 83 142 142 142 F F F 29 580-599 84 84 84 143 143 143 F F F Max gen: 85, Max flush: 144 Begin loop 43 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:29.518890531Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1480,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:29.527079863Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1480,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:29.5273715Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1480} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:29.527397036Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1480} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:29.527407674Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1480} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:29.527419323Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1480} {"msg":"Crucible 7ef2afcb-bcca-40e4-a173-8786d832ab87 has session id: beae0708-733e-4656-bbac-e72858649aac","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:29.527794167Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1480,"session_id":"beae0708-733e-4656-bbac-e72858649aac"} {"msg":"Upstairs opts: Upstairs UUID: 7ef2afcb-bcca-40e4-a173-8786d832ab87, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:29.527825063Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1480,"session_id":"beae0708-733e-4656-bbac-e72858649aac"} {"msg":"Crucible stats registered with UUID: 7ef2afcb-bcca-40e4-a173-8786d832ab87","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:29.527836101Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1480,"session_id":"beae0708-733e-4656-bbac-e72858649aac"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:56:29.52784538Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1480,"session_id":"beae0708-733e-4656-bbac-e72858649aac"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:29.527969432Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1480,"":"io task","client":"2","":"downstairs","session_id":"beae0708-733e-4656-bbac-e72858649aac"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:29.528000887Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1480,"":"io task","client":"0","":"downstairs","session_id":"beae0708-733e-4656-bbac-e72858649aac"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:29.528012065Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1480,"":"io task","client":"1","":"downstairs","session_id":"beae0708-733e-4656-bbac-e72858649aac"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:29.528174491Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1480,"task":"crutest"} {"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:29.528189119Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1480} {"msg":"7ef2afcb-bcca-40e4-a173-8786d832ab87 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:29.528221404Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1480,"session_id":"beae0708-733e-4656-bbac-e72858649aac"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:29.528231023Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1480,"client":"0","":"downstairs","session_id":"beae0708-733e-4656-bbac-e72858649aac"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:29.528241041Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1480,"client":"1","":"downstairs","session_id":"beae0708-733e-4656-bbac-e72858649aac"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:29.52825089Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1480,"client":"2","":"downstairs","session_id":"beae0708-733e-4656-bbac-e72858649aac"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:29.528262948Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1480,"":"io task","client":"2","":"downstairs","session_id":"beae0708-733e-4656-bbac-e72858649aac"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:29.528273287Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1480,"":"io task","client":"0","":"downstairs","session_id":"beae0708-733e-4656-bbac-e72858649aac"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:29.528283405Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1480,"":"io task","client":"1","":"downstairs","session_id":"beae0708-733e-4656-bbac-e72858649aac"} {"msg":"ds_connection connected from Ok(127.0.0.1:52242)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:29.528337517Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1480,"":"io task","client":"2","":"downstairs","session_id":"beae0708-733e-4656-bbac-e72858649aac"} {"msg":"ds_connection connected from Ok(127.0.0.1:44574)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:29.528371452Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1480,"":"io task","client":"1","":"downstairs","session_id":"beae0708-733e-4656-bbac-e72858649aac"} {"msg":"ds_connection connected from Ok(127.0.0.1:34520)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:29.52838318Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1480,"":"io task","client":"0","":"downstairs","session_id":"beae0708-733e-4656-bbac-e72858649aac"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:29.529056101Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1480,"client":"0","":"downstairs","session_id":"beae0708-733e-4656-bbac-e72858649aac"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:29.529075888Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1480,"client":"2","":"downstairs","session_id":"beae0708-733e-4656-bbac-e72858649aac"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:29.529091026Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1480,"client":"1","":"downstairs","session_id":"beae0708-733e-4656-bbac-e72858649aac"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:29.529381853Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1480,"session_id":"beae0708-733e-4656-bbac-e72858649aac"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:29.529391552Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1480,"session_id":"beae0708-733e-4656-bbac-e72858649aac"} {"msg":"[0]R flush_numbers[0..12]: [143, 143, 133, 141, 140, 143, 139, 143, 143, 138, 135, 143]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:29.529430526Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1480,"":"downstairs","":"downstairs","session_id":"beae0708-733e-4656-bbac-e72858649aac"} {"msg":"[0]R generation[0..12]: [84, 84, 78, 82, 82, 84, 81, 84, 84, 80, 80, 84]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:29.529441654Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1480,"":"downstairs","":"downstairs","session_id":"beae0708-733e-4656-bbac-e72858649aac"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:29.529452733Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1480,"":"downstairs","":"downstairs","session_id":"beae0708-733e-4656-bbac-e72858649aac"} {"msg":"[1]R flush_numbers[0..12]: [143, 143, 133, 141, 140, 143, 139, 143, 143, 138, 135, 143]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:29.529464971Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1480,"":"downstairs","":"downstairs","session_id":"beae0708-733e-4656-bbac-e72858649aac"} {"msg":"[1]R generation[0..12]: [84, 84, 78, 82, 82, 84, 81, 84, 84, 80, 80, 84]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:29.529475269Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1480,"":"downstairs","":"downstairs","session_id":"beae0708-733e-4656-bbac-e72858649aac"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:29.529486378Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1480,"":"downstairs","":"downstairs","session_id":"beae0708-733e-4656-bbac-e72858649aac"} {"msg":"[2]R flush_numbers[0..12]: [143, 143, 133, 141, 140, 143, 139, 143, 143, 138, 135, 143]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:29.529497306Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1480,"":"downstairs","":"downstairs","session_id":"beae0708-733e-4656-bbac-e72858649aac"} {"msg":"[2]R generation[0..12]: [84, 84, 78, 82, 82, 84, 81, 84, 84, 80, 80, 84]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:29.529507765Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1480,"":"downstairs","":"downstairs","session_id":"beae0708-733e-4656-bbac-e72858649aac"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:29.529518803Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1480,"":"downstairs","":"downstairs","session_id":"beae0708-733e-4656-bbac-e72858649aac"} {"msg":"Max found gen is 86","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:29.529528402Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1480,"":"downstairs","":"downstairs","session_id":"beae0708-733e-4656-bbac-e72858649aac"} {"msg":"Generation requested: 86 >= found:86","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:29.52953817Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1480,"":"downstairs","":"downstairs","session_id":"beae0708-733e-4656-bbac-e72858649aac"} {"msg":"Next flush: 145","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:29.529547729Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1480,"":"downstairs","":"downstairs","session_id":"beae0708-733e-4656-bbac-e72858649aac"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:29.529557327Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1480,"":"downstairs","":"downstairs","session_id":"beae0708-733e-4656-bbac-e72858649aac"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:29.529567226Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1480,"session_id":"beae0708-733e-4656-bbac-e72858649aac"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:29.529576034Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1480,"client":"0","":"downstairs","session_id":"beae0708-733e-4656-bbac-e72858649aac"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:29.529585663Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1480,"client":"1","":"downstairs","session_id":"beae0708-733e-4656-bbac-e72858649aac"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:29.529595332Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1480,"client":"2","":"downstairs","session_id":"beae0708-733e-4656-bbac-e72858649aac"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:29.52960527Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1480,"session_id":"beae0708-733e-4656-bbac-e72858649aac"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:29.529615619Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1480,"session_id":"beae0708-733e-4656-bbac-e72858649aac"} {"msg":"7ef2afcb-bcca-40e4-a173-8786d832ab87 is now active with session: beae0708-733e-4656-bbac-e72858649aac","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:29.529625617Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1480,"session_id":"beae0708-733e-4656-bbac-e72858649aac"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:29.529634396Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1480,"session_id":"beae0708-733e-4656-bbac-e72858649aac"} {"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:29.529643515Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1480} {"msg":"Activated sub_volume 7ef2afcb-bcca-40e4-a173-8786d832ab87","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:29.529651563Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1480} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:29.529659662Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1480,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:29.529984564Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1480,"task":"crutest"} 01/30 Write block 305 len 12288 data: 7 8 6 02/30 Read block 534 len 8192 03/30 Write block 460 len 12288 data: 5 5 6 04/30 Flush 05/30 Write block 254 len 24576 data: 8 8 8 8 9 7 06/30 Write block 6 len 12288 data: 6 5 5 07/30 Write block 263 len 8192 data: 8 7 08/30 Write block 375 len 36864 data: 7 7 7 5 5 3 4 5 4 09/30 Read block 590 len 36864 10/30 Write block 278 len 12288 data: 10 7 7 11/30 Read block 469 len 28672 12/30 Read block 453 len 20480 13/30 Write block 112 len 24576 data: 10 9 10 10 9 9 14/30 Read block 52 len 24576 15/30 Write block 232 len 28672 data: 5 6 5 6 6 6 6 16/30 Read block 528 len 28672 17/30 Write block 580 len 24576 data: 6 10 10 10 11 10 18/30 Write block 338 len 20480 data: 9 10 8 7 5 19/30 Write block 40 len 40960 data: 13 13 12 13 13 12 12 11 9 7 20/30 Read block 113 len 24576 21/30 Read block 488 len 20480 22/30 Read block 456 len 16384 23/30 Read block 596 len 16384 24/30 Write block 382 len 28672 data: 6 5 4 6 7 7 6 25/30 Write block 230 len 12288 data: 5 4 6 26/30 Read block 385 len 36864 27/30 Write block 100 len 36864 data: 12 13 9 10 8 9 6 7 7 28/30 Write block 221 len 40960 data: 7 7 5 5 4 4 4 5 5 6 29/30 Read block 290 len 20480 30/30 Read block 428 len 24576 ---------------------------------------------------------------- Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Write 6 Done Done Done false 1005 Acked Write 3 Done Done Done false 1006 Acked Write 2 Done Done Done false 1007 Acked Write 9 Done Done Done false 1008 Acked Read 9 Done Done Done false 1009 Acked Write 3 Done Sent Done false 1010 Acked Read 7 Done Sent Done false 1011 Acked Read 5 Done Sent Done false 1012 Acked Write 6 Done Sent Done false 1013 Acked Read 6 Done Sent Done false 1014 Acked Write 7 Done Sent Done false 1015 Acked Read 7 Done Sent Done false 1016 Acked Write 6 Done Sent Done false 1017 Acked Write 5 Done Sent Done false 1018 Acked Write 10 Done Sent Done false 1019 Acked Read 6 Done Sent Done false 1020 Acked Read 5 Done Sent Done false 1021 Acked Read 4 Done Sent Done false 1022 Acked Read 4 Done Sent Done false 1023 Acked Write 7 Done Sent Done false 1024 Acked Write 3 Done Sent Done false 1025 Acked Read 9 Done Sent Done false 1026 Acked Write 9 Done Sent Done false 1027 Acked Write 10 Done Sent Done false 1028 Acked Read 5 Done Sent Done false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 21 1 22 Done 26 5 25 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1003)) Some(JobId(1003)) Some(JobId(1003)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:56:29.538447246Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1480} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:29.53862363Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1480,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 84 84 84 143 143 143 T T T 1 020-039 84 84 84 143 143 143 F F F 2 040-059 78 78 78 133 133 133 T F T <--- 3 060-079 82 82 82 141 141 141 F F F 4 080-099 82 82 82 140 140 140 F F F 5 100-119 84 84 84 143 143 143 T F T <--- 6 120-139 81 81 81 139 139 139 F F F 7 140-159 84 84 84 143 143 143 F F F 8 160-179 84 84 84 143 143 143 F F F 9 180-199 80 80 80 138 138 138 F F F 10 200-219 80 80 80 135 135 135 F F F 11 220-239 84 84 84 143 143 143 T F T <--- 12 240-259 84 84 84 143 143 143 T T T 13 260-279 76 76 76 131 131 131 T T T 14 280-299 85 85 85 144 144 144 T F T <--- 15 300-319 86 86 86 145 145 145 F F F 16 320-339 79 79 79 134 134 134 T F T <--- 17 340-359 82 82 82 140 140 140 T F T <--- 18 360-379 84 84 84 143 143 143 T T T 19 380-399 85 85 85 144 144 144 T T T 20 400-419 85 85 85 144 144 144 F F F 21 420-439 85 85 85 144 144 144 F F F 22 440-459 85 85 85 144 144 144 F F F 23 460-479 86 86 86 145 145 145 F F F 24 480-499 82 82 82 140 140 140 F F F 25 500-519 82 82 82 141 141 141 F F F 26 520-539 82 82 82 141 141 141 F F F 27 540-559 84 84 84 143 143 143 F F F 28 560-579 83 83 83 142 142 142 F F F 29 580-599 84 84 84 143 143 143 T F T <--- Max gen: 86, Max flush: 145 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 43 Verifying data now No change after verify Loop: 43 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 146 146 146 F F F 1 020-039 84 84 84 143 143 143 F F F 2 040-059 87 87 87 146 146 146 F F F 3 060-079 82 82 82 141 141 141 F F F 4 080-099 82 82 82 140 140 140 F F F 5 100-119 87 87 87 146 146 146 F F F 6 120-139 81 81 81 139 139 139 F F F 7 140-159 84 84 84 143 143 143 F F F 8 160-179 84 84 84 143 143 143 F F F 9 180-199 80 80 80 138 138 138 F F F 10 200-219 80 80 80 135 135 135 F F F 11 220-239 87 87 87 146 146 146 F F F 12 240-259 87 87 87 146 146 146 F F F 13 260-279 87 87 87 146 146 146 F F F 14 280-299 87 87 87 146 146 146 F F F 15 300-319 86 86 86 145 145 145 F F F 16 320-339 87 87 87 146 146 146 F F F 17 340-359 87 87 87 146 146 146 F F F 18 360-379 87 87 87 146 146 146 F F F 19 380-399 87 87 87 146 146 146 F F F 20 400-419 85 85 85 144 144 144 F F F 21 420-439 85 85 85 144 144 144 F F F 22 440-459 85 85 85 144 144 144 F F F 23 460-479 86 86 86 145 145 145 F F F 24 480-499 82 82 82 140 140 140 F F F 25 500-519 82 82 82 141 141 141 F F F 26 520-539 82 82 82 141 141 141 F F F 27 540-559 84 84 84 143 143 143 F F F 28 560-579 83 83 83 142 142 142 F F F 29 580-599 87 87 87 146 146 146 F F F Max gen: 87, Max flush: 146 Begin loop 44 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:33.795959531Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1488,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:33.804020942Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1488,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:33.804325967Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1488} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:33.804356372Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1488} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:33.804365031Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1488} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:33.804376679Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1488} {"msg":"Crucible 2bd915df-ef06-4551-b4bf-5dd074cbedfa has session id: f535ab98-1e6a-48d4-a341-df57c3e41c75","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:33.804748374Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1488,"session_id":"f535ab98-1e6a-48d4-a341-df57c3e41c75"} {"msg":"Upstairs opts: Upstairs UUID: 2bd915df-ef06-4551-b4bf-5dd074cbedfa, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:33.804771021Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1488,"session_id":"f535ab98-1e6a-48d4-a341-df57c3e41c75"} {"msg":"Crucible stats registered with UUID: 2bd915df-ef06-4551-b4bf-5dd074cbedfa","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:33.804782859Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1488,"session_id":"f535ab98-1e6a-48d4-a341-df57c3e41c75"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:56:33.804791838Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1488,"session_id":"f535ab98-1e6a-48d4-a341-df57c3e41c75"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:33.804932887Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1488,"":"io task","client":"2","":"downstairs","session_id":"f535ab98-1e6a-48d4-a341-df57c3e41c75"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:33.804949365Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1488,"":"io task","client":"0","":"downstairs","session_id":"f535ab98-1e6a-48d4-a341-df57c3e41c75"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:33.804959253Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1488,"":"io task","client":"1","":"downstairs","session_id":"f535ab98-1e6a-48d4-a341-df57c3e41c75"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:33.80511645Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1488,"task":"crutest"} {"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:33.805133218Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1488} {"msg":"2bd915df-ef06-4551-b4bf-5dd074cbedfa active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:33.805172512Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1488,"session_id":"f535ab98-1e6a-48d4-a341-df57c3e41c75"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:33.8051819Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1488,"client":"0","":"downstairs","session_id":"f535ab98-1e6a-48d4-a341-df57c3e41c75"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:33.805191749Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1488,"client":"1","":"downstairs","session_id":"f535ab98-1e6a-48d4-a341-df57c3e41c75"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:33.805201397Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1488,"client":"2","":"downstairs","session_id":"f535ab98-1e6a-48d4-a341-df57c3e41c75"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:33.805210986Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1488,"":"io task","client":"2","":"downstairs","session_id":"f535ab98-1e6a-48d4-a341-df57c3e41c75"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:33.805221125Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1488,"":"io task","client":"0","":"downstairs","session_id":"f535ab98-1e6a-48d4-a341-df57c3e41c75"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:33.805234893Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1488,"":"io task","client":"1","":"downstairs","session_id":"f535ab98-1e6a-48d4-a341-df57c3e41c75"} {"msg":"ds_connection connected from Ok(127.0.0.1:55575)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:33.805260639Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1488,"":"io task","client":"0","":"downstairs","session_id":"f535ab98-1e6a-48d4-a341-df57c3e41c75"} {"msg":"ds_connection connected from Ok(127.0.0.1:55395)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:33.805271527Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1488,"":"io task","client":"1","":"downstairs","session_id":"f535ab98-1e6a-48d4-a341-df57c3e41c75"} {"msg":"ds_connection connected from Ok(127.0.0.1:35319)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:33.805372412Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1488,"":"io task","client":"2","":"downstairs","session_id":"f535ab98-1e6a-48d4-a341-df57c3e41c75"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:33.805844333Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1488,"client":"0","":"downstairs","session_id":"f535ab98-1e6a-48d4-a341-df57c3e41c75"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:33.805866119Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1488,"client":"1","":"downstairs","session_id":"f535ab98-1e6a-48d4-a341-df57c3e41c75"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:33.805972704Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1488,"client":"2","":"downstairs","session_id":"f535ab98-1e6a-48d4-a341-df57c3e41c75"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:33.80613613Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1488,"session_id":"f535ab98-1e6a-48d4-a341-df57c3e41c75"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:33.806148988Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1488,"session_id":"f535ab98-1e6a-48d4-a341-df57c3e41c75"} {"msg":"[0]R flush_numbers[0..12]: [146, 143, 146, 141, 140, 146, 139, 143, 143, 138, 135, 146]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:33.806256912Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1488,"":"downstairs","":"downstairs","session_id":"f535ab98-1e6a-48d4-a341-df57c3e41c75"} {"msg":"[0]R generation[0..12]: [87, 84, 87, 82, 82, 87, 81, 84, 84, 80, 80, 87]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:33.80626839Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1488,"":"downstairs","":"downstairs","session_id":"f535ab98-1e6a-48d4-a341-df57c3e41c75"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:33.806279528Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1488,"":"downstairs","":"downstairs","session_id":"f535ab98-1e6a-48d4-a341-df57c3e41c75"} {"msg":"[1]R flush_numbers[0..12]: [146, 143, 146, 141, 140, 146, 139, 143, 143, 138, 135, 146]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:33.806290227Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1488,"":"downstairs","":"downstairs","session_id":"f535ab98-1e6a-48d4-a341-df57c3e41c75"} {"msg":"[1]R generation[0..12]: [87, 84, 87, 82, 82, 87, 81, 84, 84, 80, 80, 87]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:33.806300645Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1488,"":"downstairs","":"downstairs","session_id":"f535ab98-1e6a-48d4-a341-df57c3e41c75"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:33.806311404Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1488,"":"downstairs","":"downstairs","session_id":"f535ab98-1e6a-48d4-a341-df57c3e41c75"} {"msg":"[2]R flush_numbers[0..12]: [146, 143, 146, 141, 140, 146, 139, 143, 143, 138, 135, 146]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:33.806322412Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1488,"":"downstairs","":"downstairs","session_id":"f535ab98-1e6a-48d4-a341-df57c3e41c75"} {"msg":"[2]R generation[0..12]: [87, 84, 87, 82, 82, 87, 81, 84, 84, 80, 80, 87]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:33.806332871Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1488,"":"downstairs","":"downstairs","session_id":"f535ab98-1e6a-48d4-a341-df57c3e41c75"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:33.806345819Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1488,"":"downstairs","":"downstairs","session_id":"f535ab98-1e6a-48d4-a341-df57c3e41c75"} {"msg":"Max found gen is 88","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:33.806355247Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1488,"":"downstairs","":"downstairs","session_id":"f535ab98-1e6a-48d4-a341-df57c3e41c75"} {"msg":"Generation requested: 88 >= found:88","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:33.806365056Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1488,"":"downstairs","":"downstairs","session_id":"f535ab98-1e6a-48d4-a341-df57c3e41c75"} {"msg":"Next flush: 147","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:33.806374454Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1488,"":"downstairs","":"downstairs","session_id":"f535ab98-1e6a-48d4-a341-df57c3e41c75"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:33.806383823Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1488,"":"downstairs","":"downstairs","session_id":"f535ab98-1e6a-48d4-a341-df57c3e41c75"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:33.806393622Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1488,"session_id":"f535ab98-1e6a-48d4-a341-df57c3e41c75"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:33.80640244Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1488,"client":"0","":"downstairs","session_id":"f535ab98-1e6a-48d4-a341-df57c3e41c75"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:33.806412249Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1488,"client":"1","":"downstairs","session_id":"f535ab98-1e6a-48d4-a341-df57c3e41c75"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:33.806422437Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1488,"client":"2","":"downstairs","session_id":"f535ab98-1e6a-48d4-a341-df57c3e41c75"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:33.806432356Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1488,"session_id":"f535ab98-1e6a-48d4-a341-df57c3e41c75"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:33.806441694Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1488,"session_id":"f535ab98-1e6a-48d4-a341-df57c3e41c75"} {"msg":"2bd915df-ef06-4551-b4bf-5dd074cbedfa is now active with session: f535ab98-1e6a-48d4-a341-df57c3e41c75","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:33.806451453Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1488,"session_id":"f535ab98-1e6a-48d4-a341-df57c3e41c75"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:33.806460202Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1488,"session_id":"f535ab98-1e6a-48d4-a341-df57c3e41c75"} {"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:33.8064693Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1488} {"msg":"Activated sub_volume 2bd915df-ef06-4551-b4bf-5dd074cbedfa","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:33.806477659Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1488} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","levelDisk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ":30,"time":"2026-07-28T18:56:33.806490867Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1488,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:33.806738881Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1488,"task":"crutest"} 01/30 Write block 488 len 8192 data: 7 5 02/30 Write block 585 len 24576 data: 11 10 10 6 6 5 03/30 Read block 56 len 4096 04/30 Read block 302 len 8192 05/30 Read block 199 len 8192 06/30 Read block 27 len 16384 07/30 Write block 256 len 20480 data: 9 9 10 8 5 08/30 Read block 52 len 40960 09/30 Read block 90 len 16384 10/30 Read block 402 len 36864 11/30 Read block 308 len 20480 12/30 Write block 30 len 4096 data: 8 13/30 Read block 564 len 4096 14/30 Read block 240 len 8192 15/30 Read block 405 len 4096 16/30 Read block 403 len 32768 17/30 Flush 18/30 Flush 19/30 Write block 15 len 8192 data: 9 9 20/30 Flush 21/30 Write block 203 len 24576 data: 5 7 8 9 13 12 22/30 Read block 59 len 36864 23/30 Read block 520 len 24576 24/30 Flush 25/30 Write block 420 len 20480 data: 4 4 7 7 8 26/30 Read block 78 len 12288 27/30 Read block 564 len 28672 28/30 Write block 213 len 16384 data: 8 8 7 5 29/30 Write block 277 len 20480 data: 10 11 8 8 6 30/30 Read block 471 len 16384 ---------------------------------------------------------------- Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Write 6 Done Done Done false 1002 Acked Read 1 Done Done Done false 1003 Acked Read 2 Done Done Sent false 1004 Acked Read 2 Done Done Sent false 1005 Acked Read 4 Done Done Sent false 1006 Acked Write 5 Done Done Sent false 1007 Acked Read 10 Done Done Sent false 1008 Acked Read 4 Done Done Sent false 1009 Acked Read 9 Done Done Sent false 1010 Acked Read 5 Done Done Sent false 1011 Acked Write 1 Done Done Sent false 1012 Acked Read 1 Done Done Sent false 1013 Acked Read 2 Done Done Sent false 1014 Acked Read 1 Done Done Sent false 1015 Acked Read 8 Done Done Sent false 1016 Acked Flush 0 Done Done Sent false 1017 Acked Flush 0 Done Done Sent false 1018 Acked Write 2 Done Done Sent false 1019 Acked Flush 0 Done Done Sent false 1020 Acked Write 6 Done Done Sent false 1021 Acked Read 9 Done Done Sent false 1022 Acked Read 6 Done Done Sent false 1023 Acked Flush 0 Done Done Sent false 1024 Acked Write 5 Done Done Sent false 1025 Acked Read 3 Done Done Sent false 1026 Acked Read 7 Done Done Sent false 1027 Acked Write 4 Done Done Sent false 1028 Acked Write 5 Done Done Sent false 1029 Acked Read 4 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 27 27 Done 30 30 3 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:56:33.8152174Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1488} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:33.815383555Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1488,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 88 88 87 149 149 146 F F F <--- 1 020-039 88 88 84 147 147 143 F F F <--- 2 040-059 87 87 87 146 146 146 F F F 3 060-079 82 82 82 141 141 141 F F F 4 080-099 82 82 82 140 140 140 F F F 5 100-119 87 87 87 146 146 146 F F F 6 120-139 81 81 81 139 139 139 F F F 7 140-159 84 84 84 143 143 143 F F F 8 160-179 84 84 84 143 143 143 F F F 9 180-199 80 80 80 138 138 138 F F F 10 200-219 88 88 80 150 150 135 T T F <--- 11 220-239 87 87 87 146 146 146 F F F 12 240-259 88 88 87 147 147 146 F F F <--- 13 260-279 88 88 87 147 147 146 T T F <--- 14 280-299 87 87 87 146 146 146 T T F <--- 15 300-319 86 86 86 145 145 145 F F F 16 320-339 87 87 87 146 146 146 F F F 17 340-359 87 87 87 146 146 146 F F F 18 360-379 87 87 87 146 146 146 F F F 19 380-399 87 87 87 146 146 146 F F F 20 400-419 85 85 85 144 144 144 F F F 21 420-439 85 85 85 144 144 144 T T F <--- 22 440-459 85 85 85 144 144 144 F F F 23 460-479 86 86 86 145 145 145 F F F 24 480-499 88 88 82 147 147 140 F F T <--- 25 500-519 82 82 82 141 141 141 F F F 26 520-539 82 82 82 141 141 141 F F F 27 540-559 84 84 84 143 143 143 F F F 28 560-579 83 83 83 142 142 142 F F F 29 580-599 88 88 87 147 147 146 F F T <--- Max gen: 88, Max flush: 150 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 44 Verifying data now No change after verify Loop: 44 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 88 88 88 149 149 149 F F F 1 020-039 88 88 88 147 147 147 F F F 2 040-059 87 87 87 146 146 146 F F F 3 060-079 82 82 82 141 141 141 F F F 4 080-099 82 82 82 140 140 140 F F F 5 100-119 87 87 87 146 146 146 F F F 6 120-139 81 81 81 139 139 139 F F F 7 140-159 84 84 84 143 143 143 F F F 8 160-179 84 84 84 143 143 143 F F F 9 180-199 80 80 80 138 138 138 F F F 10 200-219 89 89 89 151 151 151 F F F 11 220-239 87 87 87 146 146 146 F F F 12 240-259 88 88 88 147 147 147 F F F 13 260-279 89 89 89 151 151 151 F F F 14 280-299 89 89 89 151 151 151 F F F 15 300-319 86 86 86 145 145 145 F F F 16 320-339 87 87 87 146 146 146 F F F 17 340-359 87 87 87 146 146 146 F F F 18 360-379 87 87 87 146 146 146 F F F 19 380-399 87 87 87 146 146 146 F F F 20 400-419 85 85 85 144 144 144 F F F 21 420-439 89 89 89 151 151 151 F F F 22 440-459 85 85 85 144 144 144 F F F 23 460-479 86 86 86 145 145 145 F F F 24 480-499 88 88 88 147 147 147 F F F 25 500-519 82 82 82 141 141 141 F F F 26 520-539 82 82 82 141 141 141 F F F 27 540-559 84 84 84 143 143 143 F F F 28 560-579 83 83 83 142 142 142 F F F 29 580-599 88 88 88 147 147 147 F F F Max gen: 89, Max flush: 151 Begin loop 45 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:38.054666999Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1496,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:38.062737329Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1496,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:38.063036414Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1496} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:38.063061401Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1496} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:38.0630699Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1496} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:38.063081688Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1496} {"msg":"Crucible 19bce821-8883-4569-9d46-06cb8e335c4e has session id: bbe6e38a-d8d3-4228-afd3-4ec3a8d6da58","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:38.063419238Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1496,"session_id":"bbe6e38a-d8d3-4228-afd3-4ec3a8d6da58"} {"msg":"Upstairs opts: Upstairs UUID: 19bce821-8883-4569-9d46-06cb8e335c4e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:38.063448394Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1496,"session_id":"bbe6e38a-d8d3-4228-afd3-4ec3a8d6da58"} {"msg":"Crucible stats registered with UUID: 19bce821-8883-4569-9d46-06cb8e335c4e","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:38.063459742Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1496,"session_id":"bbe6e38a-d8d3-4228-afd3-4ec3a8d6da58"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:56:38.063468971Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1496,"session_id":"bbe6e38a-d8d3-4228-afd3-4ec3a8d6da58"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:38.06360769Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1496,"":"io task","client":"2","":"downstairs","session_id":"bbe6e38a-d8d3-4228-afd3-4ec3a8d6da58"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:38.063629897Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1496,"":"io task","client":"0","":"downstairs","session_id":"bbe6e38a-d8d3-4228-afd3-4ec3a8d6da58"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:38.063640165Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1496,"":"io task","client":"1","":"downstairs","session_id":"bbe6e38a-d8d3-4228-afd3-4ec3a8d6da58"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:38.063766087Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1496,"task":"crutest"} {"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:38.063780265Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1496} {"msg":"19bce821-8883-4569-9d46-06cb8e335c4e active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:38.063821029Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1496,"session_id":"bbe6e38a-d8d3-4228-afd3-4ec3a8d6da58"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:38.063830637Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1496,"client":"0","":"downstairs","session_id":"bbe6e38a-d8d3-4228-afd3-4ec3a8d6da58"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:38.063840486Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1496,"client":"1","":"downstairs","session_id":"bbe6e38a-d8d3-4228-afd3-4ec3a8d6da58"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:38.063850054Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1496,"client":"2","":"downstairs","session_id":"bbe6e38a-d8d3-4228-afd3-4ec3a8d6da58"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:38.063859803Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1496,"":"io task","client":"2","":"downstairs","session_id":"bbe6e38a-d8d3-4228-afd3-4ec3a8d6da58"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:38.063888569Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1496,"":"io task","client":"0","":"downstairs","session_id":"bbe6e38a-d8d3-4228-afd3-4ec3a8d6da58"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:38.063909046Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1496,"":"io task","client":"1","":"downstairs","session_id":"bbe6e38a-d8d3-4228-afd3-4ec3a8d6da58"} {"msg":"ds_connection connected from Ok(127.0.0.1:64661)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:38.063938381Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1496,"":"io task","client":"0","":"downstairs","session_id":"bbe6e38a-d8d3-4228-afd3-4ec3a8d6da58"} {"msg":"ds_connection connected from Ok(127.0.0.1:56514)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:38.06394961Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1496,"":"io task","client":"1","":"downstairs","session_id":"bbe6e38a-d8d3-4228-afd3-4ec3a8d6da58"} {"msg":"ds_connection connected from Ok(127.0.0.1:54661)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:38.063964358Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1496,"":"io task","client":"2","":"downstairs","session_id":"bbe6e38a-d8d3-4228-afd3-4ec3a8d6da58"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:38.064655546Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1496,"client":"1","":"downstairs","session_id":"bbe6e38a-d8d3-4228-afd3-4ec3a8d6da58"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:38.064675423Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1496,"client":"0","":"downstairs","session_id":"bbe6e38a-d8d3-4228-afd3-4ec3a8d6da58"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:38.064717496Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1496,"client":"2","":"downstairs","session_id":"bbe6e38a-d8d3-4228-afd3-4ec3a8d6da58"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:38.065022571Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1496,"session_id":"bbe6e38a-d8d3-4228-afd3-4ec3a8d6da58"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:38.065045438Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1496,"session_id":"bbe6e38a-d8d3-4228-afd3-4ec3a8d6da58"} {"msg":"[0]R flush_numbers[0..12]: [149, 147, 146, 141, 140, 146, 139, 143, 143, 138, 151, 146]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:38.065059676Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1496,"":"downstairs","":"downstairs","session_id":"bbe6e38a-d8d3-4228-afd3-4ec3a8d6da58"} {"msg":"[0]R generation[0..12]: [88, 88, 87, 82, 82, 87, 81, 84, 84, 80, 89, 87]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:38.065070714Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1496,"":"downstairs","":"downstairs","session_id":"bbe6e38a-d8d3-4228-afd3-4ec3a8d6da58"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:38.065081573Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1496,"":"downstairs","":"downstairs","session_id":"bbe6e38a-d8d3-4228-afd3-4ec3a8d6da58"} {"msg":"[1]R flush_numbers[0..12]: [149, 147, 146, 141, 140, 146, 139, 143, 143, 138, 151, 146]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:38.065092451Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1496,"":"downstairs","":"downstairs","session_id":"bbe6e38a-d8d3-4228-afd3-4ec3a8d6da58"} {"msg":"[1]R generation[0..12]: [88, 88, 87, 82, 82, 87, 81, 84, 84, 80, 89, 87]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:38.06510287Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1496,"":"downstairs","":"downstairs","session_id":"bbe6e38a-d8d3-4228-afd3-4ec3a8d6da58"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:38.065114288Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1496,"":"downstairs","":"downstairs","session_id":"bbe6e38a-d8d3-4228-afd3-4ec3a8d6da58"} {"msg":"[2]R flush_numbers[0..12]: [149, 147, 146, 141, 140, 146, 139, 143, 143, 138, 151, 146]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:38.065125186Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1496,"":"downstairs","":"downstairs","session_id":"bbe6e38a-d8d3-4228-afd3-4ec3a8d6da58"} {"msg":"[2]R generation[0..12]: [88, 88, 87, 82, 82, 87, 81, 84, 84, 80, 89, 87]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:38.065135695Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1496,"":"downstairs","":"downstairs","session_id":"bbe6e38a-d8d3-4228-afd3-4ec3a8d6da58"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:38.065146543Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1496,"":"downstairs","":"downstairs","session_id":"bbe6e38a-d8d3-4228-afd3-4ec3a8d6da58"} {"msg":"Max found gen is 90","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:38.065156142Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1496,"":"downstairs","":"downstairs","session_id":"bbe6e38a-d8d3-4228-afd3-4ec3a8d6da58"} {"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:38.06516588Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1496,"":"downstairs","":"downstairs","session_id":"bbe6e38a-d8d3-4228-afd3-4ec3a8d6da58"} {"msg":"Next flush: 152","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:38.065175419Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1496,"":"downstairs","":"downstairs","session_id":"bbe6e38a-d8d3-4228-afd3-4ec3a8d6da58"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:38.065184848Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1496,"":"downstairs","":"downstairs","session_id":"bbe6e38a-d8d3-4228-afd3-4ec3a8d6da58"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:38.065194486Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1496,"session_id":"bbe6e38a-d8d3-4228-afd3-4ec3a8d6da58"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:38.065203335Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1496,"client":"0","":"downstairs","session_id":"bbe6e38a-d8d3-4228-afd3-4ec3a8d6da58"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:38.065214423Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1496,"client":"1","":"downstairs","session_id":"bbe6e38a-d8d3-4228-afd3-4ec3a8d6da58"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:38.065223962Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1496,"client":"2","":"downstairs","session_id":"bbe6e38a-d8d3-4228-afd3-4ec3a8d6da58"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:38.06523383Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1496,"session_id":"bbe6e38a-d8d3-4228-afd3-4ec3a8d6da58"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:38.065243159Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1496,"session_id":"bbe6e38a-d8d3-4228-afd3-4ec3a8d6da58"} {"msg":"19bce821-8883-4569-9d46-06cb8e335c4e is now active with session: bbe6e38a-d8d3-4228-afd3-4ec3a8d6da58","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:38.065253357Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1496,"session_id":"bbe6e38a-d8d3-4228-afd3-4ec3a8d6da58"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:38.065262056Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1496,"session_id":"bbe6e38a-d8d3-4228-afd3-4ec3a8d6da58"} {"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:38.065271255Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1496} {"msg":"Activated sub_volume 19bce821-8883-4569-9d46-06cb8e335c4e","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:38.065279484Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1496} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:38.065287702Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1496,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:38.065605375Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1496,"task":"crutest"} 01/30 Write block 344 len 4096 data: 5 02/30 Read block 583 len 20480 03/30 Read block 269 len 8192 04/30 Write block 305 len 28672 data: 8 9 7 7 10 11 14 05/30 Read block 190 len 12288 06/30 Read block 297 len 24576 07/30 Flush 08/30 Write block 295 len 4096 data: 4 09/30 Read block 24 len 12288 10/30 Read block 588 len 20480 11/30 Read block 321 len 20480 12/30 Read block 542 len 12288 13/30 Read block 202 len 4096 14/30 Read block 452 len 20480 15/30 Flush 16/30 Write block 139 len 24576 data: 8 10 10 9 8 10 17/30 Read block 114 len 36864 18/30 Read block 231 len 20480 19/30 Write block 321 len 24576 data: 7 9 10 10 11 11 20/30 Read block 40 len 32768 21/30 Flush 22/30 Write block 372 len 20480 data: 9 7 7 8 8 23/30 Read block 122 len 16384 24/30 Flush 25/30 Write block 84 len 4096 data: 6 26/30 Read block 440 len 32768 27/30 Write block 433 len 8192 data: 7 8 28/30 Read block 236 len 8192 29/30 Read block 237 len 36864 30/30 Write block 358 len 12288 data: 10 8 9 ---------------------------------------------------------------- Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Read 5 Done Done Done false 1002 Acked Read 2 Done Done Done false 1003 Acked Write 7 Done Done Done false 1004 Acked Read 3 Done Done Done false 1005 Acked Read 6 Done Done Sent false 1006 Acked Flush 0 Done Done Sent false 1007 Acked Write 1 Done Done Sent false 1008 Acked Read 3 Done Done Sent false 1009 Acked Read 5 Done Done Sent false 1010 Acked Read 5 Done Done Sent false 1011 Acked Read 3 Done Done Sent false 1012 Acked Read 1 Done Done Sent false 1013 Acked Read 5 Done Done Sent false 1014 Acked Flush 0 Done Done Sent false 1015 Acked Write 6 Done Done Sent false 1016 Acked Read 9 Done Done Sent false 1017 Acked Read 5 Done Done Sent false 1018 Acked Write 6 Done Done Sent false 1019 Acked Read 8 Done Done Sent false 1020 Acked Flush 0 Done Done Sent false 1021 Acked Write 5 Done Done Sent false 1022 Acked Read 4 Done Done Sent false 1023 Acked Flush 0 Done Done Sent false 1024 Acked Write 1 Done Done Sent false 1025 Acked Read 8 Done Done Sent false 1026 Acked Write 2 Done Done Sent false 1027 Acked Read 2 Done Done Sent false 1028 Acked Read 9 Done Done Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 25 27 Done 29 29 5 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:56:38.074852531Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1496} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:38.075056011Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1496,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 88 88 88 149 149 149 F F F 1 020-039 88 88 88 147 147 147 F F F 2 040-059 87 87 87 146 146 146 F F F 3 060-079 82 82 82 141 141 141 F F F 4 080-099 82 82 82 140 140 140 T T F <--- 5 100-119 87 87 87 146 146 146 F F F 6 120-139 90 90 81 154 154 139 F F F <--- 7 140-159 90 90 84 154 154 143 F F F <--- 8 160-179 84 84 84 143 143 143 F F F 9 180-199 80 80 80 138 138 138 F F F 10 200-219 89 89 89 151 151 151 F F F 11 220-239 87 87 87 146 146 146 F F F 12 240-259 88 88 88 147 147 147 F F F 13 260-279 89 89 89 151 151 151 F F F 14 280-299 90 90 89 153 153 151 F F F <--- 15 300-319 90 90 86 152 152 145 F F T <--- 16 320-339 90 90 87 154 154 146 F F F <--- 17 340-359 90 90 87 152 152 146 T T T <--- 18 360-379 90 90 87 155 155 146 T T F <--- 19 380-399 87 87 87 146 146 146 F F F 20 400-419 85 85 85 144 144 144 F F F 21 420-439 89 89 89 151 151 151 T T F <--- 22 440-459 85 85 85 144 144 144 F F F 23 460-479 86 86 86 145 145 145 F F F 24 480-499 88 88 88 147 147 147 F F F 25 500-519 82 82 82 141 141 141 F F F 26 520-539 82 82 82 141 141 141 F F F 27 540-559 84 84 84 143 143 143 F F F 28 560-579 83 83 83 142 142 142 F F F 29 580-599 88 88 88 147 147 147 F F F Max gen: 90, Max flush: 155 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 45 Verifying data now No change after verify Loop: 45 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 88 88 88 149 149 149 F F F 1 020-039 88 88 88 147 147 147 F F F 2 040-059 87 87 87 146 146 146 F F F 3 060-079 82 82 82 141 141 141 F F F 4 080-099 91 91 91 156 156 156 F F F 5 100-119 87 87 87 146 146 146 F F F 6 120-139 90 90 90 154 154 154 F F F 7 140-159 90 90 90 154 154 154 F F F 8 160-179 84 84 84 143 143 143 F F F 9 180-199 80 80 80 138 138 138 F F F 10 200-219 89 89 89 151 151 151 F F F 11 220-239 87 87 87 146 146 146 F F F 12 240-259 88 88 88 147 147 147 F F F 13 260-279 89 89 89 151 151 151 F F F 14 280-299 90 90 90 153 153 153 F F F 15 300-319 90 90 90 152 152 152 F F F 16 320-339 90 90 90 154 154 154 F F F 17 340-359 91 91 91 156 156 156 F F F 18 360-379 91 91 91 156 156 156 F F F 19 380-399 87 87 87 146 146 146 F F F 20 400-419 85 85 85 144 144 144 F F F 21 420-439 91 91 91 156 156 156 F F F 22 440-459 85 85 85 144 144 144 F F F 23 460-479 86 86 86 145 145 145 F F F 24 480-499 88 88 88 147 147 147 F F F 25 500-519 82 82 82 141 141 141 F F F 26 520-539 82 82 82 141 141 141 F F F 27 540-559 84 84 84 143 143 143 F F F 28 560-579 83 83 83 142 142 142 F F F 29 580-599 88 88 88 147 147 147 F F F Max gen: 91, Max flush: 156 Begin loop 46 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:42.316434607Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1504,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:42.324459713Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1504,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:42.324744241Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1504} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:42.324771577Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1504} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:42.324780246Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1504} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:42.324791864Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1504} {"msg":"Crucible c7a9afb9-5363-48d2-a7c7-3eb8b38a3673 has session id: 066494d6-dccf-4ebf-94c9-58bf8b41d31a","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:42.325118086Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1504,"session_id":"066494d6-dccf-4ebf-94c9-58bf8b41d31a"} {"msg":"Upstairs opts: Upstairs UUID: c7a9afb9-5363-48d2-a7c7-3eb8b38a3673, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:42.325148681Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1504,"session_id":"066494d6-dccf-4ebf-94c9-58bf8b41d31a"} {"msg":"Crucible stats registered with UUID: c7a9afb9-5363-48d2-a7c7-3eb8b38a3673","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:42.32515972Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1504,"session_id":"066494d6-dccf-4ebf-94c9-58bf8b41d31a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:56:42.325169218Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1504,"session_id":"066494d6-dccf-4ebf-94c9-58bf8b41d31a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:42.325299639Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1504,"":"io task","client":"1","":"downstairs","session_id":"066494d6-dccf-4ebf-94c9-58bf8b41d31a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:42.325316446Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1504,"":"io task","client":"0","":"downstairs","session_id":"066494d6-dccf-4ebf-94c9-58bf8b41d31a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:42.325326615Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1504,"":"io task","client":"2","":"downstairs","session_id":"066494d6-dccf-4ebf-94c9-58bf8b41d31a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:42.325477983Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1504,"task":"crutest"} {"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:42.325501879Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1504} {"msg":"c7a9afb9-5363-48d2-a7c7-3eb8b38a3673 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:42.325518177Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1504,"session_id":"066494d6-dccf-4ebf-94c9-58bf8b41d31a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:42.325526985Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1504,"client":"0","":"downstairs","session_id":"066494d6-dccf-4ebf-94c9-58bf8b41d31a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:42.325536754Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1504,"client":"1","":"downstairs","session_id":"066494d6-dccf-4ebf-94c9-58bf8b41d31a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:42.325546452Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1504,"client":"2","":"downstairs","session_id":"066494d6-dccf-4ebf-94c9-58bf8b41d31a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:42.325556071Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1504,"":"io task","client":"2","":"downstairs","session_id":"066494d6-dccf-4ebf-94c9-58bf8b41d31a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:42.32556595Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1504,"":"io task","client":"0","":"downstairs","session_id":"066494d6-dccf-4ebf-94c9-58bf8b41d31a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:42.325576098Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1504,"":"io task","client":"1","":"downstairs","session_id":"066494d6-dccf-4ebf-94c9-58bf8b41d31a"} {"msg":"ds_connection connected from Ok(127.0.0.1:51881)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:42.325641768Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1504,"":"io task","client":"1","":"downstairs","session_id":"066494d6-dccf-4ebf-94c9-58bf8b41d31a"} {"msg":"ds_connection connected from Ok(127.0.0.1:49774)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:42.325666995Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1504,"":"io task","client":"0","":"downstairs","session_id":"066494d6-dccf-4ebf-94c9-58bf8b41d31a"} {"msg":"ds_connection connected from Ok(127.0.0.1:46939)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:42.325678383Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1504,"":"io task","client":"2","":"downstairs","session_id":"066494d6-dccf-4ebf-94c9-58bf8b41d31a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:42.326272305Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1504,"client":"0","":"downstairs","session_id":"066494d6-dccf-4ebf-94c9-58bf8b41d31a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:42.326361982Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1504,"client":"2","":"downstairs","session_id":"066494d6-dccf-4ebf-94c9-58bf8b41d31a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:42.326400806Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1504,"client":"1","":"downstairs","session_id":"066494d6-dccf-4ebf-94c9-58bf8b41d31a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:42.326553404Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1504,"session_id":"066494d6-dccf-4ebf-94c9-58bf8b41d31a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:42.326618094Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1504,"session_id":"066494d6-dccf-4ebf-94c9-58bf8b41d31a"} {"msg":"[0]R flush_numbers[0..12]: [149, 147, 146, 141, 156, 146, 154, 154, 143, 138, 151, 146]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:42.326703932Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1504,"":"downstairs","":"downstairs","session_id":"066494d6-dccf-4ebf-94c9-58bf8b41d31a"} {"msg":"[0]R generation[0..12]: [88, 88, 87, 82, 91, 87, 90, 90, 84, 80, 89, 87]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:42.32671517Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1504,"":"downstairs","":"downstairs","session_id":"066494d6-dccf-4ebf-94c9-58bf8b41d31a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:42.326726078Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1504,"":"downstairs","":"downstairs","session_id":"066494d6-dccf-4ebf-94c9-58bf8b41d31a"} {"msg":"[1]R flush_numbers[0..12]: [149, 147, 146, 141, 156, 146, 154, 154, 143, 138, 151, 146]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:42.326736757Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1504,"":"downstairs","":"downstairs","session_id":"066494d6-dccf-4ebf-94c9-58bf8b41d31a"} {"msg":"[1]R generation[0..12]: [88, 88, 87, 82, 91, 87, 90, 90, 84, 80, 89, 87]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:42.326747145Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1504,"":"downstairs","":"downstairs","session_id":"066494d6-dccf-4ebf-94c9-58bf8b41d31a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:42.326758104Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1504,"":"downstairs","":"downstairs","session_id":"066494d6-dccf-4ebf-94c9-58bf8b41d31a"} {"msg":"[2]R flush_numbers[0..12]: [149, 147, 146, 141, 156, 146, 154, 154, 143, 138, 151, 146]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:42.326770542Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1504,"":"downstairs","":"downstairs","session_id":"066494d6-dccf-4ebf-94c9-58bf8b41d31a"} {"msg":"[2]R generation[0..12]: [88, 88, 87, 82, 91, 87, 90, 90, 84, 80, 89, 87]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:42.32678082Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1504,"":"downstairs","":"downstairs","session_id":"066494d6-dccf-4ebf-94c9-58bf8b41d31a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:42.326791699Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1504,"":"downstairs","":"downstairs","session_id":"066494d6-dccf-4ebf-94c9-58bf8b41d31a"} {"msg":"Max found gen is 92","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:42.326801087Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1504,"":"downstairs","":"downstairs","session_id":"066494d6-dccf-4ebf-94c9-58bf8b41d31a"} {"msg":"Generation requested: 92 >= found:92","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:42.326810546Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1504,"":"downstairs","":"downstairs","session_id":"066494d6-dccf-4ebf-94c9-58bf8b41d31a"} {"msg":"Next flush: 157","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:42.326819965Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1504,"":"downstairs","":"downstairs","session_id":"066494d6-dccf-4ebf-94c9-58bf8b41d31a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:42.326829333Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1504,"":"downstairs","":"downstairs","session_id":"066494d6-dccf-4ebf-94c9-58bf8b41d31a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:42.326839092Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1504,"session_id":"066494d6-dccf-4ebf-94c9-58bf8b41d31a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:42.32684777Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1504,"client":"0","":"downstairs","session_id":"066494d6-dccf-4ebf-94c9-58bf8b41d31a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:42.326857469Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1504,"client":"1","":"downstairs","session_id":"066494d6-dccf-4ebf-94c9-58bf8b41d31a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:42.326867128Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1504,"client":"2","":"downstairs","session_id":"066494d6-dccf-4ebf-94c9-58bf8b41d31a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:42.326877266Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1504,"session_id":"066494d6-dccf-4ebf-94c9-58bf8b41d31a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:42.326886325Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1504,"session_id":"066494d6-dccf-4ebf-94c9-58bf8b41d31a"} {"msg":"c7a9afb9-5363-48d2-a7c7-3eb8b38a3673 is now active with session: 066494d6-dccf-4ebf-94c9-58bf8b41d31a","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:42.326896193Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1504,"session_id":"066494d6-dccf-4ebf-94c9-58bf8b41d31a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:42.326904942Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1504,"session_id":"066494d6-dccf-4ebf-94c9-58bf8b41d31a"} {"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:42.32691544Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1504} {"msg":"Activated sub_volume c7a9afb9-5363-48d2-a7c7-3eb8b38a3673","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:42.326923369Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1504} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:42.326931508Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1504,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:42.327227504Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1504,"task":"crutest"} 01/30 Write block 507 len 40960 data: 4 3 3 7 7 8 10 10 10 8 02/30 Read block 2 len 8192 03/30 Flush 04/30 Write block 55 len 28672 data: 6 8 6 6 4 5 4 05/30 Write block 451 len 12288 data: 12 11 11 06/30 Read block 108 len 12288 07/30 Write block 367 len 12288 data: 10 9 8 08/30 Write block 56 len 4096 data: 9 09/30 Read block 505 len 24576 10/30 Write block 277 len 4096 data: 11 11/30 Read block 93 len 36864 12/30 Write block 177 len 16384 data: 7 7 6 6 13/30 Write block 546 len 28672 data: 8 6 6 7 7 6 8 14/30 Write block 569 len 40960 data: 4 5 5 5 6 8 7 6 6 7 15/30 Write block 523 len 24576 data: 5 4 3 3 4 5 16/30 Read block 289 len 20480 17/30 Write block 485 len 16384 data: 5 4 5 8 18/30 Read block 593 len 12288 19/30 Read block 233 len 8192 20/30 Write block 6 len 12288 data: 7 6 6 21/30 Write block 562 len 12288 data: 7 7 8 22/30 Write block 340 len 12288 data: 9 8 6 23/30 Write block 120 len 20480 data: 7 9 7 8 5 24/30 Flush 25/30 Write block 93 len 36864 data: 9 7 8 8 7 8 8 13 14 26/30 Flush 27/30 Write block 251 len 16384 data: 7 7 8 9 28/30 Write block 584 len 24576 data: 12 12 11 11 7 7 29/30 Write block 9 len 20480 data: 4 4 3 5 5 30/30 Read block 452 len 32768 ---------------------------------------------------------------- Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Read 2 Done Done Done false 1002 Acked Flush 0 Done Sent Done false 1003 Acked Write 7 Done Sent Done false 1004 Acked Write 3 Done Sent Done false 1005 Acked Read 3 Done Sent Done false 1006 Acked Write 3 Done Sent Done false 1007 Acked Write 1 Done Sent Done false 1008 Acked Read 6 Done Sent Done false 1009 Acked Write 1 Done Sent Done false 1010 Acked Read 9 Done Sent Done false 1011 Acked Write 4 Done Sent Done false 1012 Acked Write 7 Done Sent Done false 1013 Acked Write 10 Done Sent Done false 1014 Acked Write 6 Done Sent Done false 1015 Acked Read 5 Done Sent Done false 1016 Acked Write 4 Done Sent Done false 1017 Acked Read 3 Done Sent Done false 1018 Acked Read 2 Done Sent Done false 1019 Acked Write 3 Done Sent Done false 1020 Acked Write 3 Done Sent Done false 1021 Acked Write 3 Done Sent Done false 1022 Acked Write 5 Done Sent Done false 1023 Acked Flush 0 Done Sent Done false 1024 Acked Write 9 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 4 Done Sent Done false 1027 Acked Write 6 Done Sent Done false 1028 Acked Write 5 Done Sent Done false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 28 1 29 Done 30 2 29 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) None Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:56:42.335071857Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1504} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:42.335234763Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1504,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 88 92 158 149 158 T F T <--- 1 020-039 88 88 88 147 147 147 F F F 2 040-059 92 87 92 158 146 158 F F F <--- 3 060-079 92 82 92 158 141 158 F F F <--- 4 080-099 92 91 92 159 156 159 F F F <--- 5 100-119 92 87 92 159 146 159 F F F <--- 6 120-139 92 90 92 158 154 158 F F F <--- 7 140-159 90 90 90 154 154 154 F F F 8 160-179 92 84 92 158 143 158 F F F <--- 9 180-199 92 80 92 158 138 158 F F F <--- 10 200-219 89 89 89 151 151 151 F F F 11 220-239 87 87 87 146 146 146 F F F 12 240-259 88 88 88 147 147 147 T F T <--- 13 260-279 92 89 92 158 151 158 F F F <--- 14 280-299 90 90 90 153 153 153 F F F 15 300-319 90 90 90 152 152 152 F F F 16 320-339 90 90 90 154 154 154 F F F 17 340-359 92 91 92 158 156 158 F F F <--- 18 360-379 92 91 92 158 156 158 F F F <--- 19 380-399 87 87 87 146 146 146 F F F 20 400-419 85 85 85 144 144 144 F F F 21 420-439 91 91 91 156 156 156 F F F 22 440-459 92 85 92 158 144 158 F F F <--- 23 460-479 86 86 86 145 145 145 F F F 24 480-499 92 88 92 158 147 158 F F F <--- 25 500-519 92 82 92 157 141 157 F T F <--- 26 520-539 92 82 92 158 141 158 F F F <--- 27 540-559 92 84 92 158 143 158 F F F <--- 28 560-579 92 83 92 158 142 158 F F F <--- 29 580-599 88 88 88 147 147 147 T F T <--- Max gen: 92, Max flush: 159 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 46 Verifying data now No change after verify Loop: 46 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 93 93 93 160 160 160 F F F 1 020-039 88 88 88 147 147 147 F F F 2 040-059 92 92 92 158 158 158 F F F 3 060-079 92 92 92 158 158 158 F F F 4 080-099 92 92 92 159 159 159 F F F 5 100-119 92 92 92 159 159 159 F F F 6 120-139 92 92 92 158 158 158 F F F 7 140-159 90 90 90 154 154 154 F F F 8 160-179 92 92 92 158 158 158 F F F 9 180-199 92 92 92 158 158 158 F F F 10 200-219 89 89 89 151 151 151 F F F 11 220-239 87 87 87 146 146 146 F F F 12 240-259 93 93 93 160 160 160 F F F 13 260-279 92 92 92 158 158 158 F F F 14 280-299 90 90 90 153 153 153 F F F 15 300-319 90 90 90 152 152 152 F F F 16 320-339 90 90 90 154 154 154 F F F 17 340-359 92 92 92 158 158 158 F F F 18 360-379 92 92 92 158 158 158 F F F 19 380-399 87 87 87 146 146 146 F F F 20 400-419 85 85 85 144 144 144 F F F 21 420-439 91 91 91 156 156 156 F F F 22 440-459 92 92 92 158 158 158 F F F 23 460-479 86 86 86 145 145 145 F F F 24 480-499 92 92 92 158 158 158 F F F 25 500-519 92 92 92 157 157 157 F F F 26 520-539 92 92 92 158 158 158 F F F 27 540-559 92 92 92 158 158 158 F F F 28 560-579 92 92 92 158 158 158 F F F 29 580-599 93 93 93 160 160 160 F F F Max gen: 93, Max flush: 160 Begin loop 47 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:46.590473214Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1512,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:46.598627141Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1512,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:46.598917618Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1512} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:46.59897455Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1512} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:46.599000156Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1512} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:46.599013384Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1512} {"msg":"Crucible ee138bab-d353-4457-add4-9368c7d8fac3 has session id: 1686695c-1952-43b1-aa4f-5265173b7992","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:46.599305791Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1512,"session_id":"1686695c-1952-43b1-aa4f-5265173b7992"} {"msg":"Upstairs opts: Upstairs UUID: ee138bab-d353-4457-add4-9368c7d8fac3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:46.599335097Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1512,"session_id":"1686695c-1952-43b1-aa4f-5265173b7992"} {"msg":"Crucible stats registered with UUID: ee138bab-d353-4457-add4-9368c7d8fac3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:46.599346355Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1512,"session_id":"1686695c-1952-43b1-aa4f-5265173b7992"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:56:46.599355793Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1512,"session_id":"1686695c-1952-43b1-aa4f-5265173b7992"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:46.599489124Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1512,"":"io task","client":"1","":"downstairs","session_id":"1686695c-1952-43b1-aa4f-5265173b7992"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:46.599505841Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1512,"":"io task","client":"0","":"downstairs","session_id":"1686695c-1952-43b1-aa4f-5265173b7992"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:46.59951591Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1512,"":"io task","client":"2","":"downstairs","session_id":"1686695c-1952-43b1-aa4f-5265173b7992"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:46.599655109Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1512,"task":"crutest"} {"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:46.599669307Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1512} {"msg":"ee138bab-d353-4457-add4-9368c7d8fac3 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:46.599709231Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1512,"session_id":"1686695c-1952-43b1-aa4f-5265173b7992"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:46.59971873Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1512,"client":"0","":"downstairs","session_id":"1686695c-1952-43b1-aa4f-5265173b7992"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:46.599728598Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1512,"client":"1","":"downstairs","session_id":"1686695c-1952-43b1-aa4f-5265173b7992"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:46.599738467Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1512,"client":"2","":"downstairs","session_id":"1686695c-1952-43b1-aa4f-5265173b7992"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:46.599748136Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1512,"":"io task","client":"2","":"downstairs","session_id":"1686695c-1952-43b1-aa4f-5265173b7992"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:46.599758254Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1512,"":"io task","client":"0","":"downstairs","session_id":"1686695c-1952-43b1-aa4f-5265173b7992"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:46.599768153Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1512,"":"io task","client":"1","":"downstairs","session_id":"1686695c-1952-43b1-aa4f-5265173b7992"} {"msg":"ds_connection connected from Ok(127.0.0.1:65395)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:46.599841072Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1512,"":"io task","client":"2","":"downstairs","session_id":"1686695c-1952-43b1-aa4f-5265173b7992"} {"msg":"ds_connection connected from Ok(127.0.0.1:58813)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:46.599871967Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1512,"":"io task","client":"1","":"downstairs","session_id":"1686695c-1952-43b1-aa4f-5265173b7992"} {"msg":"ds_connection connected from Ok(127.0.0.1:63756)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:46.599883546Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1512,"":"io task","client":"0","":"downstairs","session_id":"1686695c-1952-43b1-aa4f-5265173b7992"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:46.600510133Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1512,"client":"0","":"downstairs","session_id":"1686695c-1952-43b1-aa4f-5265173b7992"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:46.600560686Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1512,"client":"1","":"downstairs","session_id":"1686695c-1952-43b1-aa4f-5265173b7992"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:46.600572274Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1512,"client":"2","":"downstairs","session_id":"1686695c-1952-43b1-aa4f-5265173b7992"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:46.600820957Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1512,"session_id":"1686695c-1952-43b1-aa4f-5265173b7992"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:46.600845854Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1512,"session_id":"1686695c-1952-43b1-aa4f-5265173b7992"} {"msg":"[0]R flush_numbers[0..12]: [160, 147, 158, 158, 159, 159, 158, 154, 158, 158, 151, 146]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:46.600901046Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1512,"":"downstairs","":"downstairs","session_id":"1686695c-1952-43b1-aa4f-5265173b7992"} {"msg":"[0]R generation[0..12]: [93, 88, 92, 92, 92, 92, 92, 90, 92, 92, 89, 87]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:46.600912904Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1512,"":"downstairs","":"downstairs","session_id":"1686695c-1952-43b1-aa4f-5265173b7992"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:46.600924142Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1512,"":"downstairs","":"downstairs","session_id":"1686695c-1952-43b1-aa4f-5265173b7992"} {"msg":"[1]R flush_numbers[0..12]: [160, 147, 158, 158, 159, 159, 158, 154, 158, 158, 151, 146]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:46.600934841Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1512,"":"downstairs","":"downstairs","session_id":"1686695c-1952-43b1-aa4f-5265173b7992"} {"msg":"[1]R generation[0..12]: [93, 88, 92, 92, 92, 92, 92, 90, 92, 92, 89, 87]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:46.600945419Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1512,"":"downstairs","":"downstairs","session_id":"1686695c-1952-43b1-aa4f-5265173b7992"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:46.600956507Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1512,"":"downstairs","":"downstairs","session_id":"1686695c-1952-43b1-aa4f-5265173b7992"} {"msg":"[2]R flush_numbers[0..12]: [160, 147, 158, 158, 159, 159, 158, 154, 158, 158, 151, 146]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:46.600967196Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1512,"":"downstairs","":"downstairs","session_id":"1686695c-1952-43b1-aa4f-5265173b7992"} {"msg":"[2]R generation[0..12]: [93, 88, 92, 92, 92, 92, 92, 90, 92, 92, 89, 87]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:46.600977584Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1512,"":"downstairs","":"downstairs","session_id":"1686695c-1952-43b1-aa4f-5265173b7992"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:46.600988793Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1512,"":"downstairs","":"downstairs","session_id":"1686695c-1952-43b1-aa4f-5265173b7992"} {"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:46.60100925Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1512,"":"downstairs","":"downstairs","session_id":"1686695c-1952-43b1-aa4f-5265173b7992"} {"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:46.601022188Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1512,"":"downstairs","":"downstairs","session_id":"1686695c-1952-43b1-aa4f-5265173b7992"} {"msg":"Next flush: 161","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:46.601031566Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1512,"":"downstairs","":"downstairs","session_id":"1686695c-1952-43b1-aa4f-5265173b7992"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:46.601041215Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1512,"":"downstairs","":"downstairs","session_id":"1686695c-1952-43b1-aa4f-5265173b7992"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:46.601051323Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1512,"session_id":"1686695c-1952-43b1-aa4f-5265173b7992"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:46.601060162Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1512,"client":"0","":"downstairs","session_id":"1686695c-1952-43b1-aa4f-5265173b7992"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:46.601070051Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1512,"client":"1","":"downstairs","session_id":"1686695c-1952-43b1-aa4f-5265173b7992"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:46.601080129Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1512,"client":"2","":"downstairs","session_id":"1686695c-1952-43b1-aa4f-5265173b7992"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:46.601090358Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1512,"session_id":"1686695c-1952-43b1-aa4f-5265173b7992"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:46.601099746Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1512,"session_id":"1686695c-1952-43b1-aa4f-5265173b7992"} {"msg":"ee138bab-d353-4457-add4-9368c7d8fac3 is now active with session: 1686695c-1952-43b1-aa4f-5265173b7992","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:46.601109605Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1512,"session_id":"1686695c-1952-43b1-aa4f-5265173b7992"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:46.601118473Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1512,"session_id":"1686695c-1952-43b1-aa4f-5265173b7992"} {"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:46.601127632Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1512} {"msg":"Activated sub_volume ee138bab-d353-4457-add4-9368c7d8fac3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:46.601135711Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1512} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:46.60114385Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1512,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:46.601417769Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1512,"task":"crutest"} 01/30 Write block 203 len 32768 data: 6 8 9 10 14 13 11 9 02/30 Write block 5 len 4096 data: 6 03/30 Read block 84 len 40960 04/30 Read block 26 len 36864 05/30 Read block 149 len 8192 06/30 Write block 171 len 36864 data: 4 5 5 5 5 7 8 8 7 07/30 Read block 210 len 40960 08/30 Write block 342 len 40960 data: 7 6 6 4 5 4 5 4 5 6 09/30 Flush 10/30 Write block 326 len 20480 data: 12 9 7 7 8 11/30 Flush 12/30 Write block 363 len 16384 data: 10 11 13 12 13/30 Write block 403 len 36864 data: 4 6 8 6 7 6 8 11 16 14/30 Write block 531 len 32768 data: 6 7 8 8 8 9 6 7 15/30 Read block 597 len 8192 16/30 Write block 75 len 20480 data: 6 6 6 6 7 17/30 Read block 125 len 20480 18/30 Read block 220 len 40960 19/30 Read block 538 len 8192 20/30 Read block 309 len 20480 21/30 Read block 546 len 12288 22/30 Write block 156 len 8192 data: 6 8 23/30 Read block 539 len 12288 24/30 Read block 272 len 20480 25/30 Read block 210 len 8192 26/30 Read block 279 len 36864 27/30 Read block 426 len 12288 28/30 Write block 114 len 4096 data: 11 29/30 Read block 538 len 24576 30/30 Read block 363 len 24576 ---------------------------------------------------------------- Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Write 1 Done Done Done false 1002 Acked Read 10 Done Done Sent false 1003 Acked Read 9 Done Done Sent false 1004 Acked Read 2 Done Done Sent false 1005 Acked Write 9 Done Done Sent false 1006 Acked Read 10 Done Done Sent false 1007 Acked Write 10 Done Done Sent false 1008 Acked Flush 0 Done Done Sent false 1009 Acked Write 5 Done Done Sent false 1010 Acked Flush 0 Done Done Sent false 1011 Acked Write 4 Done Done Sent false 1012 Acked Write 9 Done Done Sent false 1013 Acked Write 8 Done Done Sent false 1014 Acked Read 2 Done Done Sent false 1015 Acked Write 5 Done Done Sent false 1016 Acked Read 5 Done Done Sent false 1017 Acked Read 10 Done Done Sent false 1018 Acked Read 2 Done Done Sent false 1019 Acked Read 5 Done Done Sent false 1020 Acked Read 3 Done Done Sent false 1021 Acked Write 2 Done Done Sent false 1022 Acked Read 3 Done Done Sent false 1023 Acked Read 5 Done Done Sent false 1024 Acked Read 2 Done Done Sent false 1025 Acked Read 9 Done Done Sent false 1026 Acked Read 3 Done Done Sent false 1027 Acked Write 1 Done Done Sent false 1028 Acked Read 6 Done Done Sent false 1029 Acked Read 6 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 28 28 Done 30 30 2 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1010)) Some(JobId(1010)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:56:46.610303178Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1512} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:46.610472803Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1512,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 94 94 93 161 161 160 F F T <--- 1 020-039 88 88 88 147 147 147 F F F 2 040-059 92 92 92 158 158 158 F F F 3 060-079 92 92 92 158 158 158 T T F <--- 4 080-099 92 92 92 159 159 159 F F F 5 100-119 92 92 92 159 159 159 T T F <--- 6 120-139 92 92 92 158 158 158 F F F 7 140-159 90 90 90 154 154 154 T T F <--- 8 160-179 94 94 92 161 161 158 F F F <--- 9 180-199 92 92 92 158 158 158 F F F 10 200-219 94 94 89 161 161 151 F F T <--- 11 220-239 87 87 87 146 146 146 F F F 12 240-259 93 93 93 160 160 160 F F F 13 260-279 92 92 92 158 158 158 F F F 14 280-299 90 90 90 153 153 153 F F F 15 300-319 90 90 90 152 152 152 F F F 16 320-339 94 94 90 162 162 154 F F F <--- 17 340-359 94 94 92 161 161 158 F F F <--- 18 360-379 92 92 92 158 158 158 T T F <--- 19 380-399 87 87 87 146 146 146 F F F 20 400-419 85 85 85 144 144 144 T T F <--- 21 420-439 91 91 91 156 156 156 F F F 22 440-459 92 92 92 158 158 158 F F F 23 460-479 86 86 86 145 145 145 F F F 24 480-499 92 92 92 158 158 158 F F F 25 500-519 92 92 92 157 157 157 F F F 26 520-539 92 92 92 158 158 158 T T F <--- 27 540-559 92 92 92 158 158 158 F F F 28 560-579 92 92 92 158 158 158 F F F 29 580-599 93 93 93 160 160 160 F F F Max gen: 94, Max flush: 162 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 47 Verifying data now No change after verify Loop: 47 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 94 94 94 161 161 161 F F F 1 020-039 88 88 88 147 147 147 F F F 2 040-059 92 92 92 158 158 158 F F F 3 060-079 95 95 95 163 163 163 F F F 4 080-099 92 92 92 159 159 159 F F F 5 100-119 95 95 95 163 163 163 F F F 6 120-139 92 92 92 158 158 158 F F F 7 140-159 95 95 95 163 163 163 F F F 8 160-179 94 94 94 161 161 161 F F F 9 180-199 92 92 92 158 158 158 F F F 10 200-219 94 94 94 161 161 161 F F F 11 220-239 87 87 87 146 146 146 F F F 12 240-259 93 93 93 160 160 160 F F F 13 260-279 92 92 92 158 158 158 F F F 14 280-299 90 90 90 153 153 153 F F F 15 300-319 90 90 90 152 152 152 F F F 16 320-339 94 94 94 162 162 162 F F F 17 340-359 94 94 94 161 161 161 F F F 18 360-379 95 95 95 163 163 163 F F F 19 380-399 87 87 87 146 146 146 F F F 20 400-419 95 95 95 163 163 163 F F F 21 420-439 91 91 91 156 156 156 F F F 22 440-459 92 92 92 158 158 158 F F F 23 460-479 86 86 86 145 145 145 F F F 24 480-499 92 92 92 158 158 158 F F F 25 500-519 92 92 92 157 157 157 F F F 26 520-539 95 95 95 163 163 163 F F F 27 540-559 92 92 92 158 158 158 F F F 28 560-579 92 92 92 158 158 158 F F F 29 580-599 93 93 93 160 160 160 F F F Max gen: 95, Max flush: 163 Begin loop 48 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:50.85611713Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1520,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:50.864205887Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1520,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:50.864498623Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1520} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:50.86452322Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1520} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:50.864531749Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1520} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:50.864543507Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1520} {"msg":"Crucible cb16ceb8-5eb6-4f6a-9c00-c6525671f422 has session id: c8bc84c0-82e2-4815-af64-63c1cdbc6ec0","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:50.864892205Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1520,"session_id":"c8bc84c0-82e2-4815-af64-63c1cdbc6ec0"} {"msg":"Upstairs opts: Upstairs UUID: cb16ceb8-5eb6-4f6a-9c00-c6525671f422, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:50.864921811Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1520,"session_id":"c8bc84c0-82e2-4815-af64-63c1cdbc6ec0"} {"msg":"Crucible stats registered with UUID: cb16ceb8-5eb6-4f6a-9c00-c6525671f422","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:50.864933029Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1520,"session_id":"c8bc84c0-82e2-4815-af64-63c1cdbc6ec0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:56:50.864942348Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1520,"session_id":"c8bc84c0-82e2-4815-af64-63c1cdbc6ec0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:50.865301085Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1520,"":"io task","client":"0","":"downstairs","session_id":"c8bc84c0-82e2-4815-af64-63c1cdbc6ec0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:50.865317843Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1520,"":"io task","client":"1","":"downstairs","session_id":"c8bc84c0-82e2-4815-af64-63c1cdbc6ec0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:50.865328231Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1520,"":"io task","client":"2","":"downstairs","session_id":"c8bc84c0-82e2-4815-af64-63c1cdbc6ec0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:50.865483368Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1520,"task":"crutest"} {"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:50.865501356Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1520} {"msg":"cb16ceb8-5eb6-4f6a-9c00-c6525671f422 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:50.865531881Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1520,"session_id":"c8bc84c0-82e2-4815-af64-63c1cdbc6ec0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:50.865543599Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1520,"client":"0","":"downstairs","session_id":"c8bc84c0-82e2-4815-af64-63c1cdbc6ec0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:50.865553458Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1520,"client":"1","":"downstairs","session_id":"c8bc84c0-82e2-4815-af64-63c1cdbc6ec0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:50.865563136Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1520,"client":"2","":"downstairs","session_id":"c8bc84c0-82e2-4815-af64-63c1cdbc6ec0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:50.865573255Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1520,"":"io task","client":"2","":"downstairs","session_id":"c8bc84c0-82e2-4815-af64-63c1cdbc6ec0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:50.865587133Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1520,"":"io task","client":"0","":"downstairs","session_id":"c8bc84c0-82e2-4815-af64-63c1cdbc6ec0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:50.86560659Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1520,"":"io task","client":"1","":"downstairs","session_id":"c8bc84c0-82e2-4815-af64-63c1cdbc6ec0"} {"msg":"ds_connection connected from Ok(127.0.0.1:62116)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:50.865667911Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1520,"":"io task","client":"2","":"downstairs","session_id":"c8bc84c0-82e2-4815-af64-63c1cdbc6ec0"} {"msg":"ds_connection connected from Ok(127.0.0.1:44875)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:50.865685518Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1520,"":"io task","client":"1","":"downstairs","session_id":"c8bc84c0-82e2-4815-af64-63c1cdbc6ec0"} {"msg":"ds_connection connected from Ok(127.0.0.1:40589)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:50.865701326Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1520,"":"io task","client":"0","":"downstairs","session_id":"c8bc84c0-82e2-4815-af64-63c1cdbc6ec0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:50.86635265Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1520,"client":"0","":"downstairs","session_id":"c8bc84c0-82e2-4815-af64-63c1cdbc6ec0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:50.866397473Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1520,"client":"2","":"downstairs","session_id":"c8bc84c0-82e2-4815-af64-63c1cdbc6ec0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:50.866474692Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1520,"client":"1","":"downstairs","session_id":"c8bc84c0-82e2-4815-af64-63c1cdbc6ec0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:50.866741872Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1520,"session_id":"c8bc84c0-82e2-4815-af64-63c1cdbc6ec0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:50.86675651Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1520,"session_id":"c8bc84c0-82e2-4815-af64-63c1cdbc6ec0"} {"msg":"[0]R flush_numbers[0..12]: [161, 147, 158, 163, 159, 163, 158, 163, 161, 158, 161, 146]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:50.866775937Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1520,"":"downstairs","":"downstairs","session_id":"c8bc84c0-82e2-4815-af64-63c1cdbc6ec0"} {"msg":"[0]R generation[0..12]: [94, 88, 92, 95, 92, 95, 92, 95, 94, 92, 94, 87]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:50.866788806Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1520,"":"downstairs","":"downstairs","session_id":"c8bc84c0-82e2-4815-af64-63c1cdbc6ec0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:50.866799894Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1520,"":"downstairs","":"downstairs","session_id":"c8bc84c0-82e2-4815-af64-63c1cdbc6ec0"} {"msg":"[1]R flush_numbers[0..12]: [161, 147, 158, 163, 159, 163, 158, 163, 161, 158, 161, 146]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:50.866810522Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1520,"":"downstairs","":"downstairs","session_id":"c8bc84c0-82e2-4815-af64-63c1cdbc6ec0"} {"msg":"[1]R generation[0..12]: [94, 88, 92, 95, 92, 95, 92, 95, 94, 92, 94, 87]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:50.866821041Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1520,"":"downstairs","":"downstairs","session_id":"c8bc84c0-82e2-4815-af64-63c1cdbc6ec0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:50.866832169Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1520,"":"downstairs","":"downstairs","session_id":"c8bc84c0-82e2-4815-af64-63c1cdbc6ec0"} {"msg":"[2]R flush_numbers[0..12]: [161, 147, 158, 163, 159, 163, 158, 163, 161, 158, 161, 146]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:50.866842598Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1520,"":"downstairs","":"downstairs","session_id":"c8bc84c0-82e2-4815-af64-63c1cdbc6ec0"} {"msg":"[2]R generation[0..12]: [94, 88, 92, 95, 92, 95, 92, 95, 94, 92, 94, 87]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:50.866852886Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1520,"":"downstairs","":"downstairs","session_id":"c8bc84c0-82e2-4815-af64-63c1cdbc6ec0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:50.866863904Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1520,"":"downstairs","":"downstairs","session_id":"c8bc84c0-82e2-4815-af64-63c1cdbc6ec0"} {"msg":"Max found gen is 96","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:50.866873273Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1520,"":"downstairs","":"downstairs","session_id":"c8bc84c0-82e2-4815-af64-63c1cdbc6ec0"} {"msg":"Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:50.866882982Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1520,"":"downstairs","":"downstairs","session_id":"c8bc84c0-82e2-4815-af64-63c1cdbc6ec0"} {"msg":"Next flush: 164","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:50.86689243Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1520,"":"downstairs","":"downstairs","session_id":"c8bc84c0-82e2-4815-af64-63c1cdbc6ec0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:50.866901739Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1520,"":"downstairs","":"downstairs","session_id":"c8bc84c0-82e2-4815-af64-63c1cdbc6ec0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:50.866911657Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1520,"session_id":"c8bc84c0-82e2-4815-af64-63c1cdbc6ec0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:50.866920546Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1520,"client":"0","":"downstairs","session_id":"c8bc84c0-82e2-4815-af64-63c1cdbc6ec0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:50.866930325Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1520,"client":"1","":"downstairs","session_id":"c8bc84c0-82e2-4815-af64-63c1cdbc6ec0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:50.866941423Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1520,"client":"2","":"downstairs","session_id":"c8bc84c0-82e2-4815-af64-63c1cdbc6ec0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:50.866951222Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1520,"session_id":"c8bc84c0-82e2-4815-af64-63c1cdbc6ec0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:50.86696049Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1520,"session_id":"c8bc84c0-82e2-4815-af64-63c1cdbc6ec0"} {"msg":"cb16ceb8-5eb6-4f6a-9c00-c6525671f422 is now active with session: c8bc84c0-82e2-4815-af64-63c1cdbc6ec0","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:50.866970449Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1520,"session_id":"c8bc84c0-82e2-4815-af64-63c1cdbc6ec0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:50.866979247Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1520,"session_id":"c8bc84c0-82e2-4815-af64-63c1cdbc6ec0"} {"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:50.866988366Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1520} {"msg":"Activated sub_volume cb16ceb8-5eb6-4f6a-9c00-c6525671f422","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:50.866996275Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1520} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:50.867004314Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1520,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:50.867277793Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1520,"task":"crutest"} 01/30 Write block 138 len 4096 data: 8 02/30 Write block 277 len 20480 data: 12 12 9 9 7 03/30 Read block 186 len 40960 04/30 Write block 552 len 12288 data: 9 8 8 05/30 Flush 06/30 Write block 188 len 16384 data: 6 8 8 13 07/30 Write block 569 len 28672 data: 5 6 6 6 7 9 8 08/30 Read block 230 len 12288 09/30 Read block 289 len 16384 10/30 Read block 306 len 16384 11/30 Read block 158 len 36864 12/30 Flush 13/30 Write block 223 len 12288 data: 6 6 5 14/30 Write block 141 len 8192 data: 11 10 15/30 Read block 5 len 32768 16/30 Read block 386 len 20480 17/30 Write block 457 len 40960 data: 10 7 7 6 6 7 6 8 8 12 18/30 Read block 295 len 16384 19/30 Read block 324 len 24576 20/30 Flush 21/30 Write block 487 len 16384 data: 6 9 6 6 22/30 Write block 48 len 40960 data: 10 8 7 5 4 4 6 7 10 7 23/30 Write block 146 len 16384 data: 8 9 9 8 24/30 Write block 60 len 40960 data: 6 5 6 5 5 4 5 7 6 6 25/30 Read block 445 len 16384 26/30 Flush 27/30 Write block 54 len 12288 data: 7 8 11 28/30 Write block 388 len 8192 data: 7 5 29/30 Read block 377 len 28672 30/30 Write block 242 len 4096 data: 7 ---------------------------------------------------------------- Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Write 5 Done Sent Done false 1002 Acked Read 10 Done Sent Done false 1003 Acked Write 3 Done Sent Done false 1004 Acked Flush 0 Done Sent Done false 1005 Acked Write 4 Done Sent Done false 1006 Acked Write 7 Done Sent Done false 1007 Acked Read 3 Done Sent Done false 1008 Acked Read 4 Done Sent Done false 1009 Acked Read 4 Done Sent Done false 1010 Acked Read 9 Done Sent Done false 1011 Acked Flush 0 Done Sent Done false 1012 Acked Write 3 Done Sent Done false 1013 Acked Write 2 Done Sent Done false 1014 Acked Read 8 Done Sent Done false 1015 Acked Read 5 Done Sent Done false 1016 Acked Write 10 Done Sent Done false 1017 Acked Read 4 Done Sent Done false 1018 Acked Read 6 Done Sent Done false 1019 Acked Flush 0 Done Sent Done false 1020 Acked Write 4 Done Sent Done false 1021 Acked Write 10 Done Sent Done false 1022 Acked Write 4 Done Sent Done false 1023 Acked Write 10 Done Sent Done false 1024 Acked Read 4 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 3 Done Sent Done false 1027 Acked Write 2 Done Sent Done false 1028 Acked Read 7 Done Sent Done false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 29 1 31 Done 29 1 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) None Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:56:50.875674225Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1520} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:50.875870366Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1520,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 94 94 94 161 161 161 F F F 1 020-039 88 88 88 147 147 147 F F F 2 040-059 96 92 96 167 158 167 T F T <--- 3 060-079 96 95 96 167 163 167 F F F <--- 4 080-099 92 92 92 159 159 159 F F F 5 100-119 95 95 95 163 163 163 F F F 6 120-139 96 92 96 164 158 164 F T F <--- 7 140-159 96 95 96 167 163 167 F F F <--- 8 160-179 94 94 94 161 161 161 F F F 9 180-199 96 92 96 165 158 165 F F F <--- 10 200-219 94 94 94 161 161 161 F F F 11 220-239 96 87 96 166 146 166 F F F <--- 12 240-259 93 93 93 160 160 160 T F T <--- 13 260-279 96 92 96 164 158 164 F F F <--- 14 280-299 96 90 96 164 153 164 F F F <--- 15 300-319 90 90 90 152 152 152 F F F 16 320-339 94 94 94 162 162 162 F F F 17 340-359 94 94 94 161 161 161 F F F 18 360-379 95 95 95 163 163 163 F F F 19 380-399 87 87 87 146 146 146 T F T <--- 20 400-419 95 95 95 163 163 163 F F F 21 420-439 91 91 91 156 156 156 F F F 22 440-459 96 92 96 166 158 166 F F F <--- 23 460-479 96 86 96 166 145 166 F F F <--- 24 480-499 96 92 96 167 158 167 F F F <--- 25 500-519 92 92 92 157 157 157 F F F 26 520-539 95 95 95 163 163 163 F F F 27 540-559 96 92 96 164 158 164 F F F <--- 28 560-579 96 92 96 165 158 165 F F F <--- 29 580-599 93 93 93 160 160 160 F F F Max gen: 96, Max flush: 167 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 48 Verifying data now No change after verify Loop: 48 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 94 94 94 161 161 161 F F F 1 020-039 88 88 88 147 147 147 F F F 2 040-059 97 97 97 168 168 168 F F F 3 060-079 96 96 96 167 167 167 F F F 4 080-099 92 92 92 159 159 159 F F F 5 100-119 95 95 95 163 163 163 F F F 6 120-139 96 96 96 164 164 164 F F F 7 140-159 96 96 96 167 167 167 F F F 8 160-179 94 94 94 161 161 161 F F F 9 180-199 96 96 96 165 165 165 F F F 10 200-219 94 94 94 161 161 161 F F F 11 220-239 96 96 96 166 166 166 F F F 12 240-259 97 97 97 168 168 168 F F F 13 260-279 96 96 96 164 164 164 F F F 14 280-299 96 96 96 164 164 164 F F F 15 300-319 90 90 90 152 152 152 F F F 16 320-339 94 94 94 162 162 162 F F F 17 340-359 94 94 94 161 161 161 F F F 18 360-379 95 95 95 163 163 163 F F F 19 380-399 97 97 97 168 168 168 F F F 20 400-419 95 95 95 163 163 163 F F F 21 420-439 91 91 91 156 156 156 F F F 22 440-459 96 96 96 166 166 166 F F F 23 460-479 96 96 96 166 166 166 F F F 24 480-499 96 96 96 167 167 167 F F F 25 500-519 92 92 92 157 157 157 F F F 26 520-539 95 95 95 163 163 163 F F F 27 540-559 96 96 96 164 164 164 F F F 28 560-579 96 96 96 165 165 165 F F F 29 580-599 93 93 93 160 160 160 F F F Max gen: 97, Max flush: 168 Begin loop 49 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:55.127792487Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1528,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:55.135906Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1528,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:55.136205995Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1528} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:55.136231462Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1528} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:55.13624035Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1528} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:55.136252018Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1528} {"msg":"Crucible 3e94d8b8-8e52-4868-aa31-289ff13c9b4a has session id: 91ebe1b5-3b53-45a3-9d91-efa2a633e7ca","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:55.136630183Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1528,"session_id":"91ebe1b5-3b53-45a3-9d91-efa2a633e7ca"} {"msg":"Upstairs opts: Upstairs UUID: 3e94d8b8-8e52-4868-aa31-289ff13c9b4a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:55.136662748Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1528,"session_id":"91ebe1b5-3b53-45a3-9d91-efa2a633e7ca"} {"msg":"Crucible stats registered with UUID: 3e94d8b8-8e52-4868-aa31-289ff13c9b4a","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:55.136673976Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1528,"session_id":"91ebe1b5-3b53-45a3-9d91-efa2a633e7ca"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:56:55.136683255Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1528,"session_id":"91ebe1b5-3b53-45a3-9d91-efa2a633e7ca"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:55.136813836Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1528,"":"io task","client":"1","":"downstairs","session_id":"91ebe1b5-3b53-45a3-9d91-efa2a633e7ca"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:55.136830823Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1528,"":"io task","client":"0","":"downstairs","session_id":"91ebe1b5-3b53-45a3-9d91-efa2a633e7ca"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:55.136841102Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1528,"":"io task","client":"2","":"downstairs","session_id":"91ebe1b5-3b53-45a3-9d91-efa2a633e7ca"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:55.136983521Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1528,"task":"crutest"} {"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:55.136999168Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1528} {"msg":"3e94d8b8-8e52-4868-aa31-289ff13c9b4a active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:55.137016236Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1528,"session_id":"91ebe1b5-3b53-45a3-9d91-efa2a633e7ca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:55.137025204Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1528,"client":"0","":"downstairs","session_id":"91ebe1b5-3b53-45a3-9d91-efa2a633e7ca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:55.137035003Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1528,"client":"1","":"downstairs","session_id":"91ebe1b5-3b53-45a3-9d91-efa2a633e7ca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:55.137044782Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1528,"client":"2","":"downstairs","session_id":"91ebe1b5-3b53-45a3-9d91-efa2a633e7ca"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:55.13705447Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1528,"":"io task","client":"2","":"downstairs","session_id":"91ebe1b5-3b53-45a3-9d91-efa2a633e7ca"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:55.137064659Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1528,"":"io task","client":"0","":"downstairs","session_id":"91ebe1b5-3b53-45a3-9d91-efa2a633e7ca"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:55.137074817Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1528,"":"io task","client":"1","":"downstairs","session_id":"91ebe1b5-3b53-45a3-9d91-efa2a633e7ca"} {"msg":"ds_connection connected from Ok(127.0.0.1:40015)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:55.13712606Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1528,"":"io task","client":"1","":"downstairs","session_id":"91ebe1b5-3b53-45a3-9d91-efa2a633e7ca"} {"msg":"ds_connection connected from Ok(127.0.0.1:45888)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:55.137137168Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1528,"":"io task","client":"2","":"downstairs","session_id":"91ebe1b5-3b53-45a3-9d91-efa2a633e7ca"} {"msg":"ds_connection connected from Ok(127.0.0.1:46888)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:55.137147726Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1528,"":"io task","client":"0","":"downstairs","session_id":"91ebe1b5-3b53-45a3-9d91-efa2a633e7ca"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:55.137821967Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1528,"client":"1","":"downstairs","session_id":"91ebe1b5-3b53-45a3-9d91-efa2a633e7ca"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:55.137841224Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1528,"client":"2","":"downstairs","session_id":"91ebe1b5-3b53-45a3-9d91-efa2a633e7ca"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:55.137852692Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1528,"client":"0","":"downstairs","session_id":"91ebe1b5-3b53-45a3-9d91-efa2a633e7ca"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:55.13813876Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1528,"session_id":"91ebe1b5-3b53-45a3-9d91-efa2a633e7ca"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:55.138176145Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1528,"session_id":"91ebe1b5-3b53-45a3-9d91-efa2a633e7ca"} {"msg":"[0]R flush_numbers[0..12]: [161, 147, 168, 167, 159, 163, 164, 167, 161, 165, 161, 166]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:55.138212679Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1528,"":"downstairs","":"downstairs","session_id":"91ebe1b5-3b53-45a3-9d91-efa2a633e7ca"} {"msg":"[0]R generation[0..12]: [94, 88, 97, 96, 92, 95, 96, 96, 94, 96, 94, 96]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:55.138223808Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1528,"":"downstairs","":"downstairs","session_id":"91ebe1b5-3b53-45a3-9d91-efa2a633e7ca"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:55.138235056Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1528,"":"downstairs","":"downstairs","session_id":"91ebe1b5-3b53-45a3-9d91-efa2a633e7ca"} {"msg":"[1]R flush_numbers[0..12]: [161, 147, 168, 167, 159, 163, 164, 167, 161, 165, 161, 166]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:55.138245864Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1528,"":"downstairs","":"downstairs","session_id":"91ebe1b5-3b53-45a3-9d91-efa2a633e7ca"} {"msg":"[1]R generation[0..12]: [94, 88, 97, 96, 92, 95, 96, 96, 94, 96, 94, 96]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:55.138256223Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1528,"":"downstairs","":"downstairs","session_id":"91ebe1b5-3b53-45a3-9d91-efa2a633e7ca"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:55.138267201Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1528,"":"downstairs","":"downstairs","session_id":"91ebe1b5-3b53-45a3-9d91-efa2a633e7ca"} {"msg":"[2]R flush_numbers[0..12]: [161, 147, 168, 167, 159, 163, 164, 167, 161, 165, 161, 166]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:55.138280439Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1528,"":"downstairs","":"downstairs","session_id":"91ebe1b5-3b53-45a3-9d91-efa2a633e7ca"} {"msg":"[2]R generation[0..12]: [94, 88, 97, 96, 92, 95, 96, 96, 94, 96, 94, 96]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:55.138290978Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1528,"":"downstairs","":"downstairs","session_id":"91ebe1b5-3b53-45a3-9d91-efa2a633e7ca"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:55.138301926Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1528,"":"downstairs","":"downstairs","session_id":"91ebe1b5-3b53-45a3-9d91-efa2a633e7ca"} {"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:55.138311595Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1528,"":"downstairs","":"downstairs","session_id":"91ebe1b5-3b53-45a3-9d91-efa2a633e7ca"} {"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:55.138321403Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1528,"":"downstairs","":"downstairs","session_id":"91ebe1b5-3b53-45a3-9d91-efa2a633e7ca"} {"msg":"Next flush: 169","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:55.138330882Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1528,"":"downstairs","":"downstairs","session_id":"91ebe1b5-3b53-45a3-9d91-efa2a633e7ca"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:55.13834047Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1528,"":"downstairs","":"downstairs","session_id":"91ebe1b5-3b53-45a3-9d91-efa2a633e7ca"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:55.138350279Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1528,"session_id":"91ebe1b5-3b53-45a3-9d91-efa2a633e7ca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:55.138359138Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1528,"client":"0","":"downstairs","session_id":"91ebe1b5-3b53-45a3-9d91-efa2a633e7ca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:55.138368846Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1528,"client":"1","":"downstairs","session_id":"91ebe1b5-3b53-45a3-9d91-efa2a633e7ca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:55.138378495Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1528,"client":"2","":"downstairs","session_id":"91ebe1b5-3b53-45a3-9d91-efa2a633e7ca"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:55.138388403Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1528,"session_id":"91ebe1b5-3b53-45a3-9d91-efa2a633e7ca"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:55.138397472Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1528,"session_id":"91ebe1b5-3b53-45a3-9d91-efa2a633e7ca"} {"msg":"3e94d8b8-8e52-4868-aa31-289ff13c9b4a is now active with session: 91ebe1b5-3b53-45a3-9d91-efa2a633e7ca","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:55.13840751Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1528,"session_id":"91ebe1b5-3b53-45a3-9d91-efa2a633e7ca"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:55.138416319Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1528,"session_id":"91ebe1b5-3b53-45a3-9d91-efa2a633e7ca"} {"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:55.138425438Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1528} {"msg":"Activated sub_volume 3e94d8b8-8e52-4868-aa31-289ff13c9b4a","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:55.138435036Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1528} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:55.138443025Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1528,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:55.138762628Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1528,"task":"crutest"} 01/30 Write block 445 len 24576 data: 8 8 8 9 11 11 02/30 Read block 518 len 12288 03/30 Write block 67 len 4096 data: 8 04/30 Read block 219 len 16384 05/30 Write block 195 len 4096 data: 10 06/30 Flush 07/30 Write block 128 len 16384 data: 3 5 4 4 08/30 Read block 43 len 20480 09/30 Write block 264 len 20480 data: 8 8 10 10 8 10/30 Read block 562 len 4096 11/30 Write block 88 len 40960 data: 10 9 9 8 8 10 8 9 9 8 12/30 Read block 333 len 40960 13/30 Read block 19 len 20480 14/30 Flush 15/30 Write block 71 len 4096 data: 7 16/30 Write block 202 len 4096 data: 4 17/30 Read block 131 len 8192 18/30 Flush 19/30 Flush 20/30 Write block 516 len 12288 data: 9 8 11 21/30 Write block 577 len 12288 data: 7 8 6 22/30 Write block 147 len 8192 data: 10 10 23/30 Read block 55 len 8192 24/30 Read block 103 len 40960 25/30 Read block 435 len 40960 26/30 Write block 24 len 4096 data: 11 27/30 Write block 229 len 16384 data: 6 7 5 7 28/30 Write block 204 len 28672 data: 9 10 11 15 14 12 10 29/30 Read block 217 len 24576 30/30 Read block 255 len 4096 ---------------------------------------------------------------- Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Read 3 Done Done Done false 1002 Acked Write 1 Done Done Done false 1003 Acked Read 4 Done Done Done false 1004 Acked Write 1 Done Sent Done false 1005 Acked Flush 0 Done Sent Done false 1006 Acked Write 4 Done Sent Done false 1007 Acked Read 5 Done Sent Done false 1008 Acked Write 5 Done Sent Done false 1009 Acked Read 1 Done Sent Done false 1010 Acked Write 10 Done Sent Done false 1011 Acked Read 10 Done Sent Done false 1012 Acked Read 5 Done Sent Done false 1013 Acked Flush 0 Done Sent Done false 1014 Acked Write 1 Done Sent Done false 1015 Acked Write 1 Done Sent Done false 1016 Acked Read 2 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Write 3 Done Sent Done false 1020 Acked Write 3 Done Sent Done false 1021 Acked Write 2 Done Sent Done false 1022 Acked Read 2 Done Sent Done false 1023 Acked Read 10 Done Sent Done false 1024 Acked Read 10 Done Sent Done false 1025 Acked Write 1 Done Sent Done false 1026 Acked Write 4 Done Sent Done false 1027 Acked Write 7 Done Sent Done false 1028 Acked Read 6 Done Sent Done false 1029 Acked Read 1 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 26 0 26 Done 30 4 30 64 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) None Some(JobId(1018)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:56:55.146564377Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1528} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:55.146734362Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1528,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 94 94 94 161 161 161 F F F 1 020-039 88 88 88 147 147 147 T F T <--- 2 040-059 97 97 97 168 168 168 F F F 3 060-079 98 96 98 171 167 171 F T F <--- 4 080-099 98 92 98 170 159 170 F F F <--- 5 100-119 95 95 95 163 163 163 F F F 6 120-139 98 96 98 170 164 170 F F F <--- 7 140-159 96 96 96 167 167 167 T F T <--- 8 160-179 94 94 94 161 161 161 F F F 9 180-199 98 96 98 169 165 169 F F F <--- 10 200-219 98 94 98 171 161 171 T F T <--- 11 220-239 96 96 96 166 166 166 T F T <--- 12 240-259 97 97 97 168 168 168 F F F 13 260-279 98 96 98 170 164 170 F F F <--- 14 280-299 96 96 96 164 164 164 F F F 15 300-319 90 90 90 152 152 152 F F F 16 320-339 94 94 94 162 162 162 F F F 17 340-359 94 94 94 161 161 161 F F F 18 360-379 95 95 95 163 163 163 F F F 19 380-399 97 97 97 168 168 168 F F F 20 400-419 95 95 95 163 163 163 F F F 21 420-439 91 91 91 156 156 156 F F F 22 440-459 98 96 98 169 166 169 F T F <--- 23 460-479 96 96 96 166 166 166 F F F 24 480-499 96 96 96 167 167 167 F F F 25 500-519 92 92 92 157 157 157 T F T <--- 26 520-539 95 95 95 163 163 163 F F F 27 540-559 96 96 96 164 164 164 F F F 28 560-579 96 96 96 165 165 165 T F T <--- 29 580-599 93 93 93 160 160 160 F F F Max gen: 98, Max flush: 171 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 49 Verifying data now No change after verify Loop: 49 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 94 94 94 161 161 161 F F F 1 020-039 99 99 99 172 172 172 F F F 2 040-059 97 97 97 168 168 168 F F F 3 060-079 98 98 98 171 171 171 F F F 4 080-099 98 98 98 170 170 170 F F F 5 100-119 95 95 95 163 163 163 F F F 6 120-139 98 98 98 170 170 170 F F F 7 140-159 99 99 99 172 172 172 F F F 8 160-179 94 94 94 161 161 161 F F F 9 180-199 98 98 98 169 169 169 F F F 10 200-219 99 99 99 172 172 172 F F F 11 220-239 99 99 99 172 172 172 F F F 12 240-259 97 97 97 168 168 168 F F F 13 260-279 98 98 98 170 170 170 F F F 14 280-299 96 96 96 164 164 164 F F F 15 300-319 90 90 90 152 152 152 F F F 16 320-339 94 94 94 162 162 162 F F F 17 340-359 94 94 94 161 161 161 F F F 18 360-379 95 95 95 163 163 163 F F F 19 380-399 97 97 97 168 168 168 F F F 20 400-419 95 95 95 163 163 163 F F F 21 420-439 91 91 91 156 156 156 F F F 22 440-459 98 98 98 169 169 169 F F F 23 460-479 96 96 96 166 166 166 F F F 24 480-499 96 96 96 167 167 167 F F F 25 500-519 99 99 99 172 172 172 F F F 26 520-539 95 95 95 163 163 163 F F F 27 540-559 96 96 96 164 164 164 F F F 28 560-579 99 99 99 172 172 172 F F F 29 580-599 93 93 93 160 160 160 F F F Max gen: 99, Max flush: 172 Begin loop 50 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:59.395411751Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1536,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:56:59.402453102Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1536,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:59.402509943Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1536,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:59.409401977Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1536,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:59.409670907Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1536} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:59.409695593Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1536} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:59.409703952Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1536} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:59.40971548Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1536} {"msg":"Crucible b9153d91-878f-445c-8ad8-bb7c587002fe has session id: 198a5322-6412-423b-b129-3fb7dd4cabac","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:59.410047911Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1536,"session_id":"198a5322-6412-423b-b129-3fb7dd4cabac"} {"msg":"Upstairs opts: Upstairs UUID: b9153d91-878f-445c-8ad8-bb7c587002fe, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:59.410077397Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1536,"session_id":"198a5322-6412-423b-b129-3fb7dd4cabac"} {"msg":"Crucible stats registered with UUID: b9153d91-878f-445c-8ad8-bb7c587002fe","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:59.410088375Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1536,"session_id":"198a5322-6412-423b-b129-3fb7dd4cabac"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:56:59.410103123Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1536,"session_id":"198a5322-6412-423b-b129-3fb7dd4cabac"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:59.410265109Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1536,"":"io task","client":"0","":"downstairs","session_id":"198a5322-6412-423b-b129-3fb7dd4cabac"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:59.410282707Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1536,"":"io task","client":"1","":"downstairs","session_id":"198a5322-6412-423b-b129-3fb7dd4cabac"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:59.410293055Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1536,"":"io task","client":"2","":"downstairs","session_id":"198a5322-6412-423b-b129-3fb7dd4cabac"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:59.410427205Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1536,"task":"crutest"} {"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:59.410444003Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1536} {"msg":"b9153d91-878f-445c-8ad8-bb7c587002fe active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:59.410458921Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1536,"session_id":"198a5322-6412-423b-b129-3fb7dd4cabac"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:59.410467769Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1536,"client":"0","":"downstairs","session_id":"198a5322-6412-423b-b129-3fb7dd4cabac"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:59.410477358Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1536,"client":"1","":"downstairs","session_id":"198a5322-6412-423b-b129-3fb7dd4cabac"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:59.410487017Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1536,"client":"2","":"downstairs","session_id":"198a5322-6412-423b-b129-3fb7dd4cabac"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:59.410496635Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1536,"":"io task","client":"2","":"downstairs","session_id":"198a5322-6412-423b-b129-3fb7dd4cabac"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:59.410506614Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1536,"":"io task","client":"0","":"downstairs","session_id":"198a5322-6412-423b-b129-3fb7dd4cabac"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:59.410516632Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1536,"":"io task","client":"1","":"downstairs","session_id":"198a5322-6412-423b-b129-3fb7dd4cabac"} {"msg":"ds_connection connected from Ok(127.0.0.1:62403)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:59.410566125Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1536,"":"io task","client":"2","":"downstairs","session_id":"198a5322-6412-423b-b129-3fb7dd4cabac"} {"msg":"ds_connection connected from Ok(127.0.0.1:46411)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:59.410581033Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1536,"":"io task","client":"1","":"downstairs","session_id":"198a5322-6412-423b-b129-3fb7dd4cabac"} {"msg":"ds_connection connected from Ok(127.0.0.1:34120)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:59.410634355Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1536,"":"io task","client":"0","":"downstairs","session_id":"198a5322-6412-423b-b129-3fb7dd4cabac"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:59.411253903Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1536,"client":"1","":"downstairs","session_id":"198a5322-6412-423b-b129-3fb7dd4cabac"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:59.411287099Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1536,"client":"2","":"downstairs","session_id":"198a5322-6412-423b-b129-3fb7dd4cabac"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:59.411298417Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1536,"client":"0","":"downstairs","session_id":"198a5322-6412-423b-b129-3fb7dd4cabac"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:59.411560388Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1536,"session_id":"198a5322-6412-423b-b129-3fb7dd4cabac"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:59.411572406Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1536,"session_id":"198a5322-6412-423b-b129-3fb7dd4cabac"} {"msg":"[0]R flush_numbers[0..12]: [161, 172, 168, 171, 170, 163, 170, 172, 161, 169, 172, 172]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:59.411640806Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1536,"":"downstairs","":"downstairs","session_id":"198a5322-6412-423b-b129-3fb7dd4cabac"} {"msg":"[0]R generation[0..12]: [94, 99, 97, 98, 98, 95, 98, 99, 94, 98, 99, 99]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:59.411652095Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1536,"":"downstairs","":"downstairs","session_id":"198a5322-6412-423b-b129-3fb7dd4cabac"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:59.411663373Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1536,"":"downstairs","":"downstairs","session_id":"198a5322-6412-423b-b129-3fb7dd4cabac"} {"msg":"[1]R flush_numbers[0..12]: [161, 172, 168, 171, 170, 163, 170, 172, 161, 169, 172, 172]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:59.411674061Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1536,"":"downstairs","":"downstairs","session_id":"198a5322-6412-423b-b129-3fb7dd4cabac"} {"msg":"[1]R generation[0..12]: [94, 99, 97, 98, 98, 95, 98, 99, 94, 98, 99, 99]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:59.41168432Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1536,"":"downstairs","":"downstairs","session_id":"198a5322-6412-423b-b129-3fb7dd4cabac"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:59.411695308Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1536,"":"downstairs","":"downstairs","session_id":"198a5322-6412-423b-b129-3fb7dd4cabac"} {"msg":"[2]R flush_numbers[0..12]: [161, 172, 168, 171, 170, 163, 170, 172, 161, 169, 172, 172]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:59.411705987Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1536,"":"downstairs","":"downstairs","session_id":"198a5322-6412-423b-b129-3fb7dd4cabac"} {"msg":"[2]R generation[0..12]: [94, 99, 97, 98, 98, 95, 98, 99, 94, 98, 99, 99]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:59.411716475Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1536,"":"downstairs","":"downstairs","session_id":"198a5322-6412-423b-b129-3fb7dd4cabac"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:59.411727544Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1536,"":"downstairs","":"downstairs","session_id":"198a5322-6412-423b-b129-3fb7dd4cabac"} {"msg":"Max found gen is 100","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:59.411737002Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1536,"":"downstairs","":"downstairs","session_id":"198a5322-6412-423b-b129-3fb7dd4cabac"} {"msg":"Generation requested: 100 >= found:100","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:59.41174837Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1536,"":"downstairs","":"downstairs","session_id":"198a5322-6412-423b-b129-3fb7dd4cabac"} {"msg":"Next flush: 173","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:59.411757749Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1536,"":"downstairs","":"downstairs","session_id":"198a5322-6412-423b-b129-3fb7dd4cabac"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:59.411767118Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1536,"":"downstairs","":"downstairs","session_id":"198a5322-6412-423b-b129-3fb7dd4cabac"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:59.411776976Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1536,"session_id":"198a5322-6412-423b-b129-3fb7dd4cabac"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:59.411785725Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1536,"client":"0","":"downstairs","session_id":"198a5322-6412-423b-b129-3fb7dd4cabac"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:59.411795474Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1536,"client":"1","":"downstairs","session_id":"198a5322-6412-423b-b129-3fb7dd4cabac"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:59.411805322Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1536,"client":"2","":"downstairs","session_id":"198a5322-6412-423b-b129-3fb7dd4cabac"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:59.411814971Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1536,"session_id":"198a5322-6412-423b-b129-3fb7dd4cabac"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:59.411824209Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1536,"session_id":"198a5322-6412-423b-b129-3fb7dd4cabac"} {"msg":"b9153d91-878f-445c-8ad8-bb7c587002fe is now active with session: 198a5322-6412-423b-b129-3fb7dd4cabac","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:59.411834168Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1536,"session_id":"198a5322-6412-423b-b129-3fb7dd4cabac"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:59.411843076Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1536,"session_id":"198a5322-6412-423b-b129-3fb7dd4cabac"} {"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:59.411852175Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1536} {"msg":"Activated sub_volume b9153d91-878f-445c-8ad8-bb7c587002fe","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:59.411860214Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1536} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:59.411868223Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1536,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:59.412183566Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1536,"task":"crutest"} 01/30 Write block 175 len 16384 data: 6 8 9 9 02/30 Read block 204 len 8192 03/30 Read block 422 len 20480 04/30 Write block 215 len 12288 data: 8 6 4 05/30 Write block 563 len 16384 data: 8 9 7 5 06/30 Write block 283 len 20480 data: 8 9 8 9 6 07/30 Write block 95 len 28672 data: 10 10 9 9 9 14 15 08/30 Read block 102 len 20480 09/30 Read block 343 len 32768 10/30 Write block 549 len 36864 data: 8 8 7 10 9 9 7 6 7 11/30 Write block 50 len 24576 data: 8 6 5 5 8 9 12/30 Read block 213 len 28672 13/30 Write block 103 len 4096 data: 11 14/30 Write block 286 len 12288 data: 10 7 6 15/30 Read block 324 len 28672 16/30 Write block 19 len 4096 data: 11 17/30 Read block 462 len 40960 18/30 Write block 150 len 20480 data: 8 8 8 9 8 19/30 Write block 256 len 36864 data: 10 10 11 9 6 6 5 9 9 20/30 Read block 385 len 8192 21/30 Read block 254 len 36864 22/30 Write block 28 len 4096 data: 10 23/30 Flush 24/30 Write block 551 len 32768 data: 8 11 10 10 8 7 8 7 25/30 Flush 26/30 Write block 477 len 36864 data: 8 10 10 10 9 9 7 5 6 27/30 Write block 380 len 16384 data: 4 5 7 6 28/30 Write block 35 len 24576 data: 9 9 10 14 14 14 29/30 Read block 583 len 40960 30/30 Read block 422 len 4096 ---------------------------------------------------------------- Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Read 2 Done Done Done false 1002 Acked Read 5 Sent Done Done false 1003 Acked Write 3 Sent Done Done false 1004 Acked Write 4 Sent Done Done false 1005 Acked Write 5 Sent Done Done false 1006 Acked Write 7 Sent Done Done false 1007 Acked Read 5 Sent Done Done false 1008 Acked Read 8 Sent Done Done false 1009 Acked Write 9 Sent Done Done false 1010 Acked Write 6 Sent Done Done false 1011 Acked Read 7 Sent Done Done false 1012 Acked Write 1 Sent Done Done false 1013 Acked Write 3 Sent Done Done false 1014 Acked Read 7 Sent Done Done false 1015 Acked Write 1 Sent Done Done false 1016 Acked Read 10 Sent Done Done false 1017 Acked Write 5 Sent Done Done false 1018 Acked Write 9 Sent Done Done false 1019 Acked Read 2 Sent Done Done false 1020 Acked Read 9 Sent Done Done false 1021 Acked Write 1 Sent Done Done false 1022 Acked Flush 0 Sent Done Done false 1023 Acked Write 8 Sent Done Done false 1024 Acked Flush 0 Sent Done Done false 1025 Acked Write 9 Sent Done Done false 1026 Acked Write 4 Sent Done Done false 1027 Acked Write 6 Sent Done Done false 1028 Acked Read 10 Sent Done Done false 1029 Acked Read 1 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 0 0 28 Done 2 30 30 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:56:59.420515787Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1536} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:56:59.420693721Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1536,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 94 100 100 161 173 173 F F F <--- 1 020-039 99 100 100 172 173 173 F T T <--- 2 040-059 97 100 100 168 173 173 F T T <--- 3 060-079 98 98 98 171 171 171 F F F 4 080-099 98 100 100 170 173 173 F F F <--- 5 100-119 95 100 100 163 173 173 F F F <--- 6 120-139 98 98 98 170 170 170 F F F 7 140-159 99 100 100 172 173 173 F F F <--- 8 160-179 94 100 100 161 173 173 T F F <--- 9 180-199 98 98 98 169 169 169 F F F 10 200-219 99 100 100 172 173 173 F F F <--- 11 220-239 99 99 99 172 172 172 F F F 12 240-259 97 100 100 168 173 173 F F F <--- 13 260-279 98 100 100 170 173 173 F F F <--- 14 280-299 96 100 100 164 173 173 F F F <--- 15 300-319 90 90 90 152 152 152 F F F 16 320-339 94 94 94 162 162 162 F F F 17 340-359 94 94 94 161 161 161 F F F 18 360-379 95 95 95 163 163 163 F F F 19 380-399 97 97 97 168 168 168 F T T <--- 20 400-419 95 95 95 163 163 163 F F F 21 420-439 91 91 91 156 156 156 F F F 22 440-459 98 98 98 169 169 169 F F F 23 460-479 96 96 96 166 166 166 F T T <--- 24 480-499 96 96 96 167 167 167 F T T <--- 25 500-519 99 99 99 172 172 172 F F F 26 520-539 95 95 95 163 163 163 F F F 27 540-559 96 100 100 164 174 174 F F F <--- 28 560-579 99 100 100 172 173 173 F F F <--- 29 580-599 93 93 93 160 160 160 F F F Max gen: 100, Max flush: 174 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 50 Verifying data now No change after verify Loop: 50 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 100 100 100 173 173 173 F F F 1 020-039 101 101 101 175 175 175 F F F 2 040-059 101 101 101 175 175 175 F F F 3 060-079 98 98 98 171 171 171 F F F 4 080-099 100 100 100 173 173 173 F F F 5 100-119 100 100 100 173 173 173 F F F 6 120-139 98 98 98 170 170 170 F F F 7 140-159 100 100 100 173 173 173 F F F 8 160-179 100 100 100 173 173 173 F F F 9 180-199 98 98 98 169 169 169 F F F 10 200-219 100 100 100 173 173 173 F F F 11 220-239 99 99 99 172 172 172 F F F 12 240-259 100 100 100 173 173 173 F F F 13 260-279 100 100 100 173 173 173 F F F 14 280-299 100 100 100 173 173 173 F F F 15 300-319 90 90 90 152 152 152 F F F 16 320-339 94 94 94 162 162 162 F F F 17 340-359 94 94 94 161 161 161 F F F 18 360-379 95 95 95 163 163 163 F F F 19 380-399 101 101 101 175 175 175 F F F 20 400-419 95 95 95 163 163 163 F F F 21 420-439 91 91 91 156 156 156 F F F 22 440-459 98 98 98 169 169 169 F F F 23 460-479 101 101 101 175 175 175 F F F 24 480-499 101 101 101 175 175 175 F F F 25 500-519 99 99 99 172 172 172 F F F 26 520-539 95 95 95 163 163 163 F F F 27 540-559 100 100 100 174 174 174 F F F 28 560-579 100 100 100 173 173 173 F F F 29 580-599 93 93 93 160 160 160 F F F Max gen: 101, Max flush: 175 Begin loop 51 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:03.682538547Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1544,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:03.690777171Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1544,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:03.691080817Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1544} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:03.691107683Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1544} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:03.691116301Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1544} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:03.69112787Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1544} {"msg":"Crucible e84ba7ef-9392-495c-93b9-907c329f8e25 has session id: 6847a2d8-8536-46c0-bcc2-6faf15d5b5f4","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:03.691486847Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1544,"session_id":"6847a2d8-8536-46c0-bcc2-6faf15d5b5f4"} {"msg":"Upstairs opts: Upstairs UUID: e84ba7ef-9392-495c-93b9-907c329f8e25, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:03.691516722Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1544,"session_id":"6847a2d8-8536-46c0-bcc2-6faf15d5b5f4"} {"msg":"Crucible stats registered with UUID: e84ba7ef-9392-495c-93b9-907c329f8e25","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:03.691527761Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1544,"session_id":"6847a2d8-8536-46c0-bcc2-6faf15d5b5f4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:57:03.691536839Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1544,"session_id":"6847a2d8-8536-46c0-bcc2-6faf15d5b5f4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:03.691676479Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1544,"":"io task","client":"2","":"downstairs","session_id":"6847a2d8-8536-46c0-bcc2-6faf15d5b5f4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:03.691707084Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1544,"":"io task","client":"0","":"downstairs","session_id":"6847a2d8-8536-46c0-bcc2-6faf15d5b5f4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:03.691718272Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1544,"":"io task","client":"1","":"downstairs","session_id":"6847a2d8-8536-46c0-bcc2-6faf15d5b5f4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:03.691845794Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1544,"task":"crutest"} {"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:03.691865021Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1544} {"msg":"e84ba7ef-9392-495c-93b9-907c329f8e25 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:03.691893107Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1544,"session_id":"6847a2d8-8536-46c0-bcc2-6faf15d5b5f4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:03.691902715Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1544,"client":"0","":"downstairs","session_id":"6847a2d8-8536-46c0-bcc2-6faf15d5b5f4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:03.691912444Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1544,"client":"1","":"downstairs","session_id":"6847a2d8-8536-46c0-bcc2-6faf15d5b5f4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:03.691922212Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1544,"client":"2","":"downstairs","session_id":"6847a2d8-8536-46c0-bcc2-6faf15d5b5f4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:03.691931921Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1544,"":"io task","client":"2","":"downstairs","session_id":"6847a2d8-8536-46c0-bcc2-6faf15d5b5f4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:03.691942199Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1544,"":"io task","client":"0","":"downstairs","session_id":"6847a2d8-8536-46c0-bcc2-6faf15d5b5f4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:03.691951978Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1544,"":"io task","client":"1","":"downstairs","session_id":"6847a2d8-8536-46c0-bcc2-6faf15d5b5f4"} {"msg":"ds_connection connected from Ok(127.0.0.1:51007)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:03.691996351Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1544,"":"io task","client":"1","":"downstairs","session_id":"6847a2d8-8536-46c0-bcc2-6faf15d5b5f4"} {"msg":"ds_connection connected from Ok(127.0.0.1:33685)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:03.692017138Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1544,"":"io task","client":"2","":"downstairs","session_id":"6847a2d8-8536-46c0-bcc2-6faf15d5b5f4"} {"msg":"ds_connection connected from Ok(127.0.0.1:35189)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:03.692027607Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1544,"":"io task","client":"0","":"downstairs","session_id":"6847a2d8-8536-46c0-bcc2-6faf15d5b5f4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:03.692724434Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1544,"client":"0","":"downstairs","session_id":"6847a2d8-8536-46c0-bcc2-6faf15d5b5f4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:03.692799523Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1544,"client":"2","":"downstairs","session_id":"6847a2d8-8536-46c0-bcc2-6faf15d5b5f4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:03.692853065Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1544,"client":"1","":"downstairs","session_id":"6847a2d8-8536-46c0-bcc2-6faf15d5b5f4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:03.693012142Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1544,"session_id":"6847a2d8-8536-46c0-bcc2-6faf15d5b5f4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:03.693053066Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1544,"session_id":"6847a2d8-8536-46c0-bcc2-6faf15d5b5f4"} {"msg":"[0]R flush_numbers[0..12]: [173, 175, 175, 171, 173, 173, 170, 173, 173, 169, 173, 172]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:03.693138173Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1544,"":"downstairs","":"downstairs","session_id":"6847a2d8-8536-46c0-bcc2-6faf15d5b5f4"} {"msg":"[0]R generation[0..12]: [100, 101, 101, 98, 100, 100, 98, 100, 100, 98, 100, 99]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:03.693153131Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1544,"":"downstairs","":"downstairs","session_id":"6847a2d8-8536-46c0-bcc2-6faf15d5b5f4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:03.693164389Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1544,"":"downstairs","":"downstairs","session_id":"6847a2d8-8536-46c0-bcc2-6faf15d5b5f4"} {"msg":"[1]R flush_numbers[0..12]: [173, 175, 175, 171, 173, 173, 170, 173, 173, 169, 173, 172]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:03.693175248Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1544,"":"downstairs","":"downstairs","session_id":"6847a2d8-8536-46c0-bcc2-6faf15d5b5f4"} {"msg":"[1]R generation[0..12]: [100, 101, 101, 98, 100, 100, 98, 100, 100, 98, 100, 99]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:03.693185746Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1544,"":"downstairs","":"downstairs","session_id":"6847a2d8-8536-46c0-bcc2-6faf15d5b5f4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:03.693196884Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1544,"":"downstairs","":"downstairs","session_id":"6847a2d8-8536-46c0-bcc2-6faf15d5b5f4"} {"msg":"[2]R flush_numbers[0..12]: [173, 175, 175, 171, 173, 173, 170, 173, 173, 169, 173, 172]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:03.693207413Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1544,"":"downstairs","":"downstairs","session_id":"6847a2d8-8536-46c0-bcc2-6faf15d5b5f4"} {"msg":"[2]R generation[0..12]: [100, 101, 101, 98, 100, 100, 98, 100, 100, 98, 100, 99]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:03.693218001Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1544,"":"downstairs","":"downstairs","session_id":"6847a2d8-8536-46c0-bcc2-6faf15d5b5f4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:03.69322918Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1544,"":"downstairs","":"downstairs","session_id":"6847a2d8-8536-46c0-bcc2-6faf15d5b5f4"} {"msg":"Max found gen is 102","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:03.693238968Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1544,"":"downstairs","":"downstairs","session_id":"6847a2d8-8536-46c0-bcc2-6faf15d5b5f4"} {"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:03.693248917Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1544,"":"downstairs","":"downstairs","session_id":"6847a2d8-8536-46c0-bcc2-6faf15d5b5f4"} {"msg":"Next flush: 176","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:03.693258465Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1544,"":"downstairs","":"downstairs","session_id":"6847a2d8-8536-46c0-bcc2-6faf15d5b5f4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:03.693267974Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1544,"":"downstairs","":"downstairs","session_id":"6847a2d8-8536-46c0-bcc2-6faf15d5b5f4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:03.693278122Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1544,"session_id":"6847a2d8-8536-46c0-bcc2-6faf15d5b5f4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:03.693287091Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1544,"client":"0","":"downstairs","session_id":"6847a2d8-8536-46c0-bcc2-6faf15d5b5f4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:03.693298749Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1544,"client":"1","":"downstairs","session_id":"6847a2d8-8536-46c0-bcc2-6faf15d5b5f4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:03.693308898Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1544,"client":"2","":"downstairs","session_id":"6847a2d8-8536-46c0-bcc2-6faf15d5b5f4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:03.693319136Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1544,"session_id":"6847a2d8-8536-46c0-bcc2-6faf15d5b5f4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:03.693328595Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1544,"session_id":"6847a2d8-8536-46c0-bcc2-6faf15d5b5f4"} {"msg":"e84ba7ef-9392-495c-93b9-907c329f8e25 is now active with session: 6847a2d8-8536-46c0-bcc2-6faf15d5b5f4","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:03.693338553Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1544,"session_id":"6847a2d8-8536-46c0-bcc2-6faf15d5b5f4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:03.693347402Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1544,"session_id":"6847a2d8-8536-46c0-bcc2-6faf15d5b5f4"} {"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:03.693356481Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1544} {"msg":"Activated sub_volume e84ba7ef-9392-495c-93b9-907c329f8e25","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:03.6933646Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1544} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:03.693373148Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1544,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:03.693654927Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1544,"task":"crutest"} 01/30 Write block 376 len 8192 data: 9 8 02/30 Write block 193 len 12288 data: 13 13 11 03/30 Read block 315 len 28672 04/30 Write block 565 len 4096 data: 8 05/30 Read block 288 len 28672 06/30 Write block 86 len 28672 data: 7 7 11 10 10 9 9 07/30 Read block 574 len 16384 08/30 Write block 476 len 16384 data: 8 9 11 11 09/30 Read block 237 len 32768 10/30 Read block 265 len 40960 11/30 Write block 17 len 16384 data: 10 10 12 13 12/30 Read block 284 len 40960 13/30 Write block 368 len 4096 data: 10 14/30 Write block 378 len 36864 data: 6 6 5 6 8 7 5 7 8 15/30 Read block 73 len 12288 16/30 Write block 254 len 24576 data: 10 9 11 11 12 10 17/30 Flush 18/30 Write block 235 len 20480 data: 7 7 7 7 5 19/30 Read block 317 len 24576 20/30 Flush 21/30 Write block 191 len 40960 data: 14 14 14 14 12 8 10 9 8 6 22/30 Flush 23/30 Write block 218 len 28672 data: 3 4 6 8 8 7 7 24/30 Read block 227 len 28672 25/30 Write block 124 len 16384 data: 6 5 5 5 26/30 Write block 324 len 20480 data: 11 12 13 10 8 27/30 Write block 482 len 36864 data: 10 8 6 7 5 7 10 7 7 28/30 Write block 207 len 16384 data: 16 15 13 11 29/30 Read block 52 len 16384 30/30 Read block 334 len 24576 ---------------------------------------------------------------- Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Write 3 Done Done Done false 1002 Acked Read 7 Done Sent Done false 1003 Acked Write 1 Done Sent Done false 1004 Acked Read 7 Done Sent Done false 1005 Acked Write 7 Done Sent Done false 1006 Acked Read 4 Done Sent Done false 1007 Acked Write 4 Done Sent Done false 1008 Acked Read 8 Done Sent Done false 1009 Acked Read 10 Done Sent Done false 1010 Acked Write 4 Done Sent Done false 1011 Acked Read 10 Done Sent Done false 1012 Acked Write 1 Done Sent Done false 1013 Acked Write 9 Done Sent Done false 1014 Acked Read 3 Done Sent Done false 1015 Acked Write 6 Done Sent Done false 1016 Acked Flush 0 Done Sent Done false 1017 Acked Write 5 Done Sent Done false 1018 Acked Read 6 Done Sent Done false 1019 Acked Flush 0 Done Sent Done false 1020 Acked Write 10 Done Sent Done false 1021 Acked Flush 0 Done Sent Done false 1022 Acked Write 7 Done Sent Done false 1023 Acked Read 7 Done Sent Done false 1024 Acked Write 4 Done Sent Done false 1025 Acked Write 5 Done Sent Done false 1026 Acked Write 9 Done Sent Done false 1027 Acked Write 4 Done Sent Done false 1028 Acked Read 4 Done Sent Done false 1029 Acked Read 6 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 28 0 29 Done 29 2 30 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) None Some(JobId(1021)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:57:03.702329667Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1544} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:03.702506971Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1544,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 100 102 176 173 176 F F F <--- 1 020-039 102 101 102 176 175 176 F F F <--- 2 040-059 101 101 101 175 175 175 F F F 3 060-079 98 98 98 171 171 171 F F F 4 080-099 102 100 102 176 173 176 F F F <--- 5 100-119 100 100 100 173 173 173 F F F 6 120-139 98 98 98 170 170 170 T F T <--- 7 140-159 100 100 100 173 173 173 F F F 8 160-179 100 100 100 173 173 173 F F F 9 180-199 102 98 102 178 169 178 F T F <--- 10 200-219 102 100 102 178 173 178 T F T <--- 11 220-239 102 99 102 177 172 177 T F T <--- 12 240-259 102 100 102 176 173 176 F F F <--- 13 260-279 100 100 100 173 173 173 F F F 14 280-299 100 100 100 173 173 173 F F F 15 300-319 90 90 90 152 152 152 F F F 16 320-339 94 94 94 162 162 162 T F T <--- 17 340-359 94 94 94 161 161 161 F F F 18 360-379 102 95 102 176 163 176 F T F <--- 19 380-399 102 101 102 176 175 176 F F F <--- 20 400-419 95 95 95 163 163 163 F F F 21 420-439 91 91 91 156 156 156 F F F 22 440-459 98 98 98 169 169 169 F F F 23 460-479 102 101 102 176 175 176 F F F <--- 24 480-499 101 101 101 175 175 175 T F T <--- 25 500-519 99 99 99 172 172 172 F F F 26 520-539 95 95 95 163 163 163 F F F 27 540-559 100 100 100 174 174 174 F F F 28 560-579 102 100 102 176 173 176 F F F <--- 29 580-599 93 93 93 160 160 160 F F F Max gen: 102, Max flush: 178 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 51 Verifying data now No change after verify Loop: 51 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 176 176 176 F F F 1 020-039 102 102 102 176 176 176 F F F 2 040-059 101 101 101 175 175 175 F F F 3 060-079 98 98 98 171 171 171 F F F 4 080-099 102 102 102 176 176 176 F F F 5 100-119 100 100 100 173 173 173 F F F 6 120-139 103 103 103 179 179 179 F F F 7 140-159 100 100 100 173 173 173 F F F 8 160-179 100 100 100 173 173 173 F F F 9 180-199 102 102 102 178 178 178 F F F 10 200-219 103 103 103 179 179 179 F F F 11 220-239 103 103 103 179 179 179 F F F 12 240-259 102 102 102 176 176 176 F F F 13 260-279 100 100 100 173 173 173 F F F 14 280-299 100 100 100 173 173 173 F F F 15 300-319 90 90 90 152 152 152 F F F 16 320-339 103 103 103 179 179 179 F F F 17 340-359 94 94 94 161 161 161 F F F 18 360-379 102 102 102 176 176 176 F F F 19 380-399 102 102 102 176 176 176 F F F 20 400-419 95 95 95 163 163 163 F F F 21 420-439 91 91 91 156 156 156 F F F 22 440-459 98 98 98 169 169 169 F F F 23 460-479 102 102 102 176 176 176 F F F 24 480-499 103 103 103 179 179 179 F F F 25 500-519 99 99 99 172 172 172 F F F 26 520-539 95 95 95 163 163 163 F F F 27 540-559 100 100 100 174 174 174 F F F 28 560-579 102 102 102 176 176 176 F F F 29 580-599 93 93 93 160 160 160 F F F Max gen: 103, Max flush: 179 Begin loop 52 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:07.959637412Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1552,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:57:07.966792346Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1552,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:07.966858766Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1552,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:07.973975946Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1552,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:07.974269093Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1552} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:07.974295099Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1552} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:07.974306148Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1552} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:07.974318016Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1552} {"msg":"Crucible f7bb3728-a88e-4951-bed4-6a32fa842eb6 has session id: f292a298-2d27-418b-b734-f3212a1983c4","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:07.974664075Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1552,"session_id":"f292a298-2d27-418b-b734-f3212a1983c4"} {"msg":"Upstairs opts: Upstairs UUID: f7bb3728-a88e-4951-bed4-6a32fa842eb6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:07.974686371Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1552,"session_id":"f292a298-2d27-418b-b734-f3212a1983c4"} {"msg":"Crucible stats registered with UUID: f7bb3728-a88e-4951-bed4-6a32fa842eb6","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:07.97469635Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1552,"session_id":"f292a298-2d27-418b-b734-f3212a1983c4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:57:07.974705689Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1552,"session_id":"f292a298-2d27-418b-b734-f3212a1983c4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:07.974862995Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1552,"":"io task","client":"2","":"downstairs","session_id":"f292a298-2d27-418b-b734-f3212a1983c4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:07.97489897Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1552,"":"io task","client":"0","":"downstairs","session_id":"f292a298-2d27-418b-b734-f3212a1983c4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:07.974910258Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1552,"":"io task","client":"1","":"downstairs","session_id":"f292a298-2d27-418b-b734-f3212a1983c4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:07.975047288Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1552,"task":"crutest"} {"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:07.975066515Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1552} {"msg":"f7bb3728-a88e-4951-bed4-6a32fa842eb6 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:07.975097211Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1552,"session_id":"f292a298-2d27-418b-b734-f3212a1983c4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:07.975106709Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1552,"client":"0","":"downstairs","session_id":"f292a298-2d27-418b-b734-f3212a1983c4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:07.975116508Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1552,"client":"1","":"downstairs","session_id":"f292a298-2d27-418b-b734-f3212a1983c4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:07.975126407Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1552,"client":"2","":"downstairs","session_id":"f292a298-2d27-418b-b734-f3212a1983c4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:07.975138285Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1552,"":"io task","client":"2","":"downstairs","session_id":"f292a298-2d27-418b-b734-f3212a1983c4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:07.975148513Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1552,"":"io task","client":"0","":"downstairs","session_id":"f292a298-2d27-418b-b734-f3212a1983c4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:07.975158542Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1552,"":"io task","client":"1","":"downstairs","session_id":"f292a298-2d27-418b-b734-f3212a1983c4"} {"msg":"ds_connection connected from Ok(127.0.0.1:63184)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:07.975190087Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1552,"":"io task","client":"0","":"downstairs","session_id":"f292a298-2d27-418b-b734-f3212a1983c4"} {"msg":"ds_connection connected from Ok(127.0.0.1:37513)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:07.975201295Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1552,"":"io task","client":"2","":"downstairs","session_id":"f292a298-2d27-418b-b734-f3212a1983c4"} {"msg":"ds_connection connected from Ok(127.0.0.1:62308)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:07.975211734Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1552,"":"io task","client":"1","":"downstairs","session_id":"f292a298-2d27-418b-b734-f3212a1983c4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:07.975877676Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1552,"client":"1","":"downstairs","session_id":"f292a298-2d27-418b-b734-f3212a1983c4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:07.975904132Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1552,"client":"2","":"downstairs","session_id":"f292a298-2d27-418b-b734-f3212a1983c4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:07.97591608Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1552,"client":"0","":"downstairs","session_id":"f292a298-2d27-418b-b734-f3212a1983c4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:07.976236193Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1552,"session_id":"f292a298-2d27-418b-b734-f3212a1983c4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:07.976260709Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1552,"session_id":"f292a298-2d27-418b-b734-f3212a1983c4"} {"msg":"[0]R flush_numbers[0..12]: [176, 176, 175, 171, 176, 173, 179, 173, 173, 178, 179, 179]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:07.976271668Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1552,"":"downstairs","":"downstairs","session_id":"f292a298-2d27-418b-b734-f3212a1983c4"} {"msg":"[0]R generation[0..12]: [102, 102, 101, 98, 102, 100, 103, 100, 100, 102, 103, 103]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:07.976282996Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1552,"":"downstairs","":"downstairs","session_id":"f292a298-2d27-418b-b734-f3212a1983c4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:07.976294174Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1552,"":"downstairs","":"downstairs","session_id":"f292a298-2d27-418b-b734-f3212a1983c4"} {"msg":"[1]R flush_numbers[0..12]: [176, 176, 175, 171, 176, 173, 179, 173, 173, 178, 179, 179]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:07.976307432Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1552,"":"downstairs","":"downstairs","session_id":"f292a298-2d27-418b-b734-f3212a1983c4"} {"msg":"[1]R generation[0..12]: [102, 102, 101, 98, 102, 100, 103, 100, 100, 102, 103, 103]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:07.976318121Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1552,"":"downstairs","":"downstairs","session_id":"f292a298-2d27-418b-b734-f3212a1983c4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:07.976329269Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1552,"":"downstairs","":"downstairs","session_id":"f292a298-2d27-418b-b734-f3212a1983c4"} {"msg":"[2]R flush_numbers[0..12]: [176, 176, 175, 171, 176, 173, 179, 173, 173, 178, 179, 179]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:07.976339858Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1552,"":"downstairs","":"downstairs","session_id":"f292a298-2d27-418b-b734-f3212a1983c4"} {"msg":"[2]R generation[0..12]: [102, 102, 101, 98, 102, 100, 103, 100, 100, 102, 103, 103]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:07.976350466Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1552,"":"downstairs","":"downstairs","session_id":"f292a298-2d27-418b-b734-f3212a1983c4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:07.976361464Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1552,"":"downstairs","":"downstairs","session_id":"f292a298-2d27-418b-b734-f3212a1983c4"} {"msg":"Max found gen is 104","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:07.976370983Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1552,"":"downstairs","":"downstairs","session_id":"f292a298-2d27-418b-b734-f3212a1983c4"} {"msg":"Generation requested: 104 >= found:104","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:07.976380861Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1552,"":"downstairs","":"downstairs","session_id":"f292a298-2d27-418b-b734-f3212a1983c4"} {"msg":"Next flush: 180","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:07.97639056Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1552,"":"downstairs","":"downstairs","session_id":"f292a298-2d27-418b-b734-f3212a1983c4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:07.976400339Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1552,"":"downstairs","":"downstairs","session_id":"f292a298-2d27-418b-b734-f3212a1983c4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:07.976410377Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1552,"session_id":"f292a298-2d27-418b-b734-f3212a1983c4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:07.976419286Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1552,"client":"0","":"downstairs","session_id":"f292a298-2d27-418b-b734-f3212a1983c4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:07.976429044Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1552,"client":"1","":"downstairs","session_id":"f292a298-2d27-418b-b734-f3212a1983c4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:07.976438743Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1552,"client":"2","":"downstairs","session_id":"f292a298-2d27-418b-b734-f3212a1983c4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:07.976448551Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1552,"session_id":"f292a298-2d27-418b-b734-f3212a1983c4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:07.97645923Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1552,"session_id":"f292a298-2d27-418b-b734-f3212a1983c4"} {"msg":"f7bb3728-a88e-4951-bed4-6a32fa842eb6 is now active with session: f292a298-2d27-418b-b734-f3212a1983c4","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:07.976469348Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1552,"session_id":"f292a298-2d27-418b-b734-f3212a1983c4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:07.976478177Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1552,"session_id":"f292a298-2d27-418b-b734-f3212a1983c4"} {"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:07.976487256Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1552} {"msg":"Activated sub_volume f7bb3728-a88e-4951-bed4-6a32fa842eb6","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:07.976495265Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1552} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:07.976503233Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1552,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:07.97679849Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1552,"task":"crutest"} 01/30 Write block 592 len 28672 data: 10 11 11 10 8 5 3 02/30 Write block 245 len 8192 data: 10 10 03/30 Write block 144 len 40960 data: 11 10 9 11 11 9 9 9 9 10 04/30 Read block 283 len 16384 05/30 Read block 288 len 32768 06/30 Read block 416 len 20480 07/30 Read block 59 len 16384 08/30 Read block 532 len 4096 09/30 Read block 558 len 8192 10/30 Write block 227 len 24576 data: 5 6 7 8 6 8 11/30 Write block 237 len 16384 data: 8 8 6 7 12/30 Read block 478 len 4096 13/30 Write block 183 len 24576 data: 5 6 5 5 7 7 14/30 Read block 145 len 28672 15/30 Read block 305 len 8192 16/30 Write block 365 len 36864 data: 14 13 11 11 9 10 8 10 8 17/30 Write block 554 len 32768 data: 11 9 8 9 8 6 6 8 18/30 Flush 19/30 Write block 133 len 28672 data: 6 10 9 8 7 9 9 20/30 Read block 466 len 4096 21/30 Read block 455 len 24576 22/30 Read block 479 len 20480 23/30 Write block 273 len 36864 data: 9 10 11 9 13 13 10 10 8 24/30 Read block 254 len 20480 25/30 Read block 427 len 8192 26/30 Write block 547 len 28672 data: 7 7 9 9 9 12 11 27/30 Write block 382 len 12288 data: 9 8 6 28/30 Read block 488 len 36864 29/30 Read block 94 len 32768 30/30 Write block 123 len 20480 data: 9 7 6 6 6 ---------------------------------------------------------------- Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Write 2 Done Done Done false 1002 Acked Write 10 Done Done Done false 1003 Acked Read 4 Done Done Done false 1004 Acked Read 8 Done Done Done false 1005 Acked Read 5 Done Done Done false 1006 Acked Read 4 Done Done Done false 1007 Acked Read 1 Done Done Done false 1008 Acked Read 2 Done Done Done false 1009 Acked Write 6 Sent Done Done false 1010 Acked Write 4 Sent Done Done false 1011 Acked Read 1 Sent Done Done false 1012 Acked Write 6 Sent Done Done false 1013 Acked Read 7 Sent Done Done false 1014 Acked Read 2 Sent Done Done false 1015 Acked Write 9 Sent Done Done false 1016 Acked Write 8 Sent Done Done false 1017 Acked Flush 0 Sent Done Done false 1018 Acked Write 7 Sent Done Done false 1019 Acked Read 1 Sent Done Done false 1020 Acked Read 6 Sent Done Done false 1021 Acked Read 5 Sent Done Done false 1022 Acked Write 9 Sent Done Done false 1023 Acked Read 5 Sent Done Done false 1024 Acked Read 2 Sent Done Done false 1025 Acked Write 7 Sent Done Done false 1026 Acked Write 3 Sent Done Done false 1027 Acked Read 9 Sent Done Done false 1028 Acked Read 8 Sent Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 21 1 1 23 Done 9 29 29 67 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:57:07.985985864Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1552} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:07.986168057Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1552,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 176 176 176 F F F 1 020-039 102 102 102 176 176 176 F F F 2 040-059 101 101 101 175 175 175 F F F 3 060-079 98 98 98 171 171 171 F F F 4 080-099 102 102 102 176 176 176 F F F 5 100-119 100 100 100 173 173 173 F F F 6 120-139 103 103 103 179 179 179 F T T <--- 7 140-159 100 104 104 173 180 180 T F F <--- 8 160-179 100 100 100 173 173 173 F F F 9 180-199 102 104 104 178 180 180 F F F <--- 10 200-219 103 103 103 179 179 179 F F F 11 220-239 103 104 104 179 180 180 F F F <--- 12 240-259 102 104 104 176 180 180 T F F <--- 13 260-279 100 100 100 173 173 173 F T T <--- 14 280-299 100 100 100 173 173 173 F T T <--- 15 300-319 90 90 90 152 152 152 F F F 16 320-339 103 103 103 179 179 179 F F F 17 340-359 94 94 94 161 161 161 F F F 18 360-379 102 104 104 176 180 180 F F F <--- 19 380-399 102 102 102 176 176 176 F T T <--- 20 400-419 95 95 95 163 163 163 F F F 21 420-439 91 91 91 156 156 156 F F F 22 440-459 98 98 98 169 169 169 F F F 23 460-479 102 102 102 176 176 176 F F F 24 480-499 103 103 103 179 179 179 F F F 25 500-519 99 99 99 172 172 172 F F F 26 520-539 95 95 95 163 163 163 F F F 27 540-559 100 104 104 174 180 180 F T T <--- 28 560-579 102 104 104 176 180 180 F F F <--- 29 580-599 93 104 104 160 180 180 T F F <--- Max gen: 104, Max flush: 180 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 52 Verifying data now No change after verify Loop: 52 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 176 176 176 F F F 1 020-039 102 102 102 176 176 176 F F F 2 040-059 101 101 101 175 175 175 F F F 3 060-079 98 98 98 171 171 171 F F F 4 080-099 102 102 102 176 176 176 F F F 5 100-119 100 100 100 173 173 173 F F F 6 120-139 105 105 105 181 181 181 F F F 7 140-159 104 104 104 180 180 180 F F F 8 160-179 100 100 100 173 173 173 F F F 9 180-199 104 104 104 180 180 180 F F F 10 200-219 103 103 103 179 179 179 F F F 11 220-239 104 104 104 180 180 180 F F F 12 240-259 104 104 104 180 180 180 F F F 13 260-279 105 105 105 181 181 181 F F F 14 280-299 105 105 105 181 181 181 F F F 15 300-319 90 90 90 152 152 152 F F F 16 320-339 103 103 103 179 179 179 F F F 17 340-359 94 94 94 161 161 161 F F F 18 360-379 104 104 104 180 180 180 F F F 19 380-399 105 105 105 181 181 181 F F F 20 400-419 95 95 95 163 163 163 F F F 21 420-439 91 91 91 156 156 156 F F F 22 440-459 98 98 98 169 169 169 F F F 23 460-479 102 102 102 176 176 176 F F F 24 480-499 103 103 103 179 179 179 F F F 25 500-519 99 99 99 172 172 172 F F F 26 520-539 95 95 95 163 163 163 F F F 27 540-559 105 105 105 181 181 181 F F F 28 560-579 104 104 104 180 180 180 F F F 29 580-599 104 104 104 180 180 180 F F F Max gen: 105, Max flush: 181 Begin loop 53 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:12.238971487Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1560,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:12.24729329Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1560,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:12.247583227Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1560} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:12.247606543Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1560} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:12.247614922Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1560} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:12.247626511Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1560} {"msg":"Crucible 720233b4-69df-4c0d-a8f6-fd44fb956c6f has session id: 5db9d56d-df70-4cc1-adbd-c012262a2d54","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:12.248012214Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1560,"session_id":"5db9d56d-df70-4cc1-adbd-c012262a2d54"} {"msg":"Upstairs opts: Upstairs UUID: 720233b4-69df-4c0d-a8f6-fd44fb956c6f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:12.248032521Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1560,"session_id":"5db9d56d-df70-4cc1-adbd-c012262a2d54"} {"msg":"Crucible stats registered with UUID: 720233b4-69df-4c0d-a8f6-fd44fb956c6f","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:12.248044549Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1560,"session_id":"5db9d56d-df70-4cc1-adbd-c012262a2d54"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:57:12.248053618Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1560,"session_id":"5db9d56d-df70-4cc1-adbd-c012262a2d54"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:12.248198376Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1560,"":"io task","client":"2","":"downstairs","session_id":"5db9d56d-df70-4cc1-adbd-c012262a2d54"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:12.248228062Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1560,"":"io task","client":"0","":"downstairs","session_id":"5db9d56d-df70-4cc1-adbd-c012262a2d54"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:12.24823894Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1560,"":"io task","client":"1","":"downstairs","session_id":"5db9d56d-df70-4cc1-adbd-c012262a2d54"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:12.248381099Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1560,"task":"crutest"} {"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:12.248394917Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1560} {"msg":"720233b4-69df-4c0d-a8f6-fd44fb956c6f active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:12.248419284Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1560,"session_id":"5db9d56d-df70-4cc1-adbd-c012262a2d54"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:12.248428752Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1560,"client":"0","":"downstairs","session_id":"5db9d56d-df70-4cc1-adbd-c012262a2d54"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:12.248438331Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1560,"client":"1","":"downstairs","session_id":"5db9d56d-df70-4cc1-adbd-c012262a2d54"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:12.248448139Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1560,"client":"2","":"downstairs","session_id":"5db9d56d-df70-4cc1-adbd-c012262a2d54"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:12.248457738Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1560,"":"io task","client":"2","":"downstairs","session_id":"5db9d56d-df70-4cc1-adbd-c012262a2d54"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:12.248467796Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1560,"":"io task","client":"0","":"downstairs","session_id":"5db9d56d-df70-4cc1-adbd-c012262a2d54"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:12.248503551Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1560,"":"io task","client":"1","":"downstairs","session_id":"5db9d56d-df70-4cc1-adbd-c012262a2d54"} {"msg":"ds_connection connected from Ok(127.0.0.1:48471)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:12.24851447Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1560,"":"io task","client":"2","":"downstairs","session_id":"5db9d56d-df70-4cc1-adbd-c012262a2d54"} {"msg":"ds_connection connected from Ok(127.0.0.1:46407)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:12.248550844Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1560,"":"io task","client":"0","":"downstairs","session_id":"5db9d56d-df70-4cc1-adbd-c012262a2d54"} {"msg":"ds_connection connected from Ok(127.0.0.1:35689)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:12.24864482Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1560,"":"io task","client":"1","":"downstairs","session_id":"5db9d56d-df70-4cc1-adbd-c012262a2d54"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:12.249147266Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1560,"client":"0","":"downstairs","session_id":"5db9d56d-df70-4cc1-adbd-c012262a2d54"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:12.249207157Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1560,"client":"2","":"downstairs","session_id":"5db9d56d-df70-4cc1-adbd-c012262a2d54"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:12.249445942Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1560,"session_id":"5db9d56d-df70-4cc1-adbd-c012262a2d54"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:12.249470349Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1560,"client":"1","":"downstairs","session_id":"5db9d56d-df70-4cc1-adbd-c012262a2d54"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:12.249485506Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1560,"session_id":"5db9d56d-df70-4cc1-adbd-c012262a2d54"} {"msg":"[0]R flush_numbers[0..12]: [176, 176, 175, 171, 176, 173, 181, 180, 173, 180, 179, 180]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:12.24979604Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1560,"":"downstairs","":"downstairs","session_id":"5db9d56d-df70-4cc1-adbd-c012262a2d54"} {"msg":"[0]R generation[0..12]: [102, 102, 101, 98, 102, 100, 105, 104, 100, 104, 103, 104]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:12.249807549Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1560,"":"downstairs","":"downstairs","session_id":"5db9d56d-df70-4cc1-adbd-c012262a2d54"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:12.249818617Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1560,"":"downstairs","":"downstairs","session_id":"5db9d56d-df70-4cc1-adbd-c012262a2d54"} {"msg":"[1]R flush_numbers[0..12]: [176, 176, 175, 171, 176, 173, 181, 180, 173, 180, 179, 180]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:12.249829376Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1560,"":"downstairs","":"downstairs","session_id":"5db9d56d-df70-4cc1-adbd-c012262a2d54"} {"msg":"[1]R generation[0..12]: [102, 102, 101, 98, 102, 100, 105, 104, 100, 104, 103, 104]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:12.249839924Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1560,"":"downstairs","":"downstairs","session_id":"5db9d56d-df70-4cc1-adbd-c012262a2d54"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:12.249851002Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1560,"":"downstairs","":"downstairs","session_id":"5db9d56d-df70-4cc1-adbd-c012262a2d54"} {"msg":"[2]R flush_numbers[0..12]: [176, 176, 175, 171, 176, 173, 181, 180, 173, 180, 179, 180]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:12.249861741Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1560,"":"downstairs","":"downstairs","session_id":"5db9d56d-df70-4cc1-adbd-c012262a2d54"} {"msg":"[2]R generation[0..12]: [102, 102, 101, 98, 102, 100, 105, 104, 100, 104, 103, 104]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:12.249872329Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1560,"":"downstairs","":"downstairs","session_id":"5db9d56d-df70-4cc1-adbd-c012262a2d54"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:12.249885907Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1560,"":"downstairs","":"downstairs","session_id":"5db9d56d-df70-4cc1-adbd-c012262a2d54"} {"msg":"Max found gen is 106","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:12.249895346Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1560,"":"downstairs","":"downstairs","session_id":"5db9d56d-df70-4cc1-adbd-c012262a2d54"} {"msg":"Generation requested: 106 >= found:106","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:12.249905204Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1560,"":"downstairs","":"downstairs","session_id":"5db9d56d-df70-4cc1-adbd-c012262a2d54"} {"msg":"Next flush: 182","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:12.249914693Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1560,"":"downstairs","":"downstairs","session_id":"5db9d56d-df70-4cc1-adbd-c012262a2d54"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:12.249924182Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1560,"":"downstairs","":"downstairs","session_id":"5db9d56d-df70-4cc1-adbd-c012262a2d54"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:12.2499338Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1560,"session_id":"5db9d56d-df70-4cc1-adbd-c012262a2d54"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:12.249942779Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1560,"client":"0","":"downstairs","session_id":"5db9d56d-df70-4cc1-adbd-c012262a2d54"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:12.249952577Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1560,"client":"1","":"downstairs","session_id":"5db9d56d-df70-4cc1-adbd-c012262a2d54"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:12.249962246Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1560,"client":"2","":"downstairs","session_id":"5db9d56d-df70-4cc1-adbd-c012262a2d54"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:12.249972024Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1560,"session_id":"5db9d56d-df70-4cc1-adbd-c012262a2d54"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:12.249981193Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1560,"session_id":"5db9d56d-df70-4cc1-adbd-c012262a2d54"} {"msg":"720233b4-69df-4c0d-a8f6-fd44fb956c6f is now active with session: 5db9d56d-df70-4cc1-adbd-c012262a2d54","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:12.249991142Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1560,"session_id":"5db9d56d-df70-4cc1-adbd-c012262a2d54"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:12.24999986Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1560,"session_id":"5db9d56d-df70-4cc1-adbd-c012262a2d54"} {"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:12.250009179Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1560} {"msg":"Activated sub_volume 720233b4-69df-4c0d-a8f6-fd44fb956c6f","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:12.250017218Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1560} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:12.250025347Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1560,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:12.25041059Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1560,"task":"crutest"} 01/30 Write block 403 len 24576 data: 5 7 9 7 8 7 02/30 Write block 475 len 36864 data: 10 9 10 12 12 11 10 11 9 03/30 Read block 55 len 28672 04/30 Read block 580 len 36864 05/30 Write block 329 len 40960 data: 8 9 8 7 6 6 6 8 9 10 06/30 Read block 381 len 12288 07/30 Read block 106 len 4096 08/30 Write block 419 len 28672 data: 5 5 5 8 8 9 9 09/30 Flush 10/30 Write block 44 len 32768 data: 14 13 13 12 11 9 9 7 11/30 Read block 394 len 20480 12/30 Read block 238 len 24576 13/30 Read block 122 len 40960 14/30 Write block 433 len 4096 data: 8 15/30 Read block 163 len 24576 16/30 Write block 408 len 40960 data: 8 9 12 17 14 15 11 11 12 7 17/30 Read block 395 len 8192 18/30 Write block 500 len 32768 data: 7 5 7 6 4 3 3 5 19/30 Read block 225 len 24576 20/30 Write block 541 len 8192 data: 7 7 21/30 Read block 360 len 40960 22/30 Read block 594 len 8192 23/30 Read block 501 len 32768 24/30 Write block 398 len 4096 data: 6 25/30 Write block 187 len 28672 data: 8 8 9 9 15 15 15 26/30 Write block 143 len 16384 data: 9 12 11 10 27/30 Read block 251 len 8192 28/30 Write block 201 len 12288 data: 4 5 7 29/30 Read block 343 len 8192 30/30 Write block 253 len 28672 data: 9 11 10 12 12 13 11 ---------------------------------------------------------------- Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Sent Done false 1001 Acked Write 9 Done Sent Done false 1002 Acked Read 7 Done Sent Done false 1003 Acked Read 9 Done Sent Done false 1004 Acked Write 10 Done Sent Done false 1005 Acked Read 3 Done Sent Done false 1006 Acked Read 1 Done Sent Done false 1007 Acked Write 7 Done Sent Done false 1008 Acked Flush 0 Done Sent Done false 1009 Acked Write 8 Done Sent Done false 1010 Acked Read 5 Done Sent Done false 1011 Acked Read 6 Done Sent Done false 1012 Acked Read 10 Done Sent Done false 1013 Acked Write 1 Done Sent Done false 1014 Acked Read 6 Done Sent Done false 1015 Acked Write 10 Done Sent Done false 1016 Acked Read 2 Done Sent Done false 1017 Acked Write 8 Done Sent Done false 1018 Acked Read 6 Done Sent Done false 1019 Acked Write 2 Done Sent Done false 1020 Acked Read 10 Done Sent Done false 1021 Acked Read 2 Done Sent Done false 1022 Acked Read 8 Done Sent Done false 1023 Acked Write 1 Done Sent Done false 1024 Acked Write 7 Done Sent Done false 1025 Acked Write 4 Done Sent Done false 1026 Acked Read 2 Done Sent Done false 1027 Acked Write 3 Done Sent Done false 1028 Acked Read 2 Done Sent Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 30 1 32 Done 29 0 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1008)) None Some(JobId(1008)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:57:12.259476602Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1560} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:12.259674803Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1560,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 176 176 176 F F F 1 020-039 102 102 102 176 176 176 F F F 2 040-059 101 101 101 175 175 175 T F T <--- 3 060-079 98 98 98 171 171 171 F F F 4 080-099 102 102 102 176 176 176 F F F 5 100-119 100 100 100 173 173 173 F F F 6 120-139 105 105 105 181 181 181 F F F 7 140-159 104 104 104 180 180 180 T F T <--- 8 160-179 100 100 100 173 173 173 F F F 9 180-199 104 104 104 180 180 180 T F T <--- 10 200-219 103 103 103 179 179 179 T F T <--- 11 220-239 104 104 104 180 180 180 F F F 12 240-259 104 104 104 180 180 180 T F T <--- 13 260-279 105 105 105 181 181 181 F F F 14 280-299 105 105 105 181 181 181 F F F 15 300-319 90 90 90 152 152 152 F F F 16 320-339 106 103 106 182 179 182 F F F <--- 17 340-359 94 94 94 161 161 161 F F F 18 360-379 104 104 104 180 180 180 F F F 19 380-399 105 105 105 181 181 181 T F T <--- 20 400-419 106 95 106 182 163 182 T F T <--- 21 420-439 106 91 106 182 156 182 T F T <--- 22 440-459 98 98 98 169 169 169 F F F 23 460-479 106 102 106 182 176 182 F F F <--- 24 480-499 106 103 106 182 179 182 F F F <--- 25 500-519 99 99 99 172 172 172 T F T <--- 26 520-539 95 95 95 163 163 163 F F F 27 540-559 105 105 105 181 181 181 T F T <--- 28 560-579 104 104 104 180 180 180 F F F 29 580-599 104 104 104 180 180 180 F F F Max gen: 106, Max flush: 182 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 53 Verifying data now No change after verify Loop: 53 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 176 176 176 F F F 1 020-039 102 102 102 176 176 176 F F F 2 040-059 107 107 107 183 183 183 F F F 3 060-079 98 98 98 171 171 171 F F F 4 080-099 102 102 102 176 176 176 F F F 5 100-119 100 100 100 173 173 173 F F F 6 120-139 105 105 105 181 181 181 F F F 7 140-159 107 107 107 183 183 183 F F F 8 160-179 100 100 100 173 173 173 F F F 9 180-199 107 107 107 183 183 183 F F F 10 200-219 107 107 107 183 183 183 F F F 11 220-239 104 104 104 180 180 180 F F F 12 240-259 107 107 107 183 183 183 F F F 13 260-279 105 105 105 181 181 181 F F F 14 280-299 105 105 105 181 181 181 F F F 15 300-319 90 90 90 152 152 152 F F F 16 320-339 106 106 106 182 182 182 F F F 17 340-359 94 94 94 161 161 161 F F F 18 360-379 104 104 104 180 180 180 F F F 19 380-399 107 107 107 183 183 183 F F F 20 400-419 107 107 107 183 183 183 F F F 21 420-439 107 107 107 183 183 183 F F F 22 440-459 98 98 98 169 169 169 F F F 23 460-479 106 106 106 182 182 182 F F F 24 480-499 106 106 106 182 182 182 F F F 25 500-519 107 107 107 183 183 183 F F F 26 520-539 95 95 95 163 163 163 F F F 27 540-559 107 107 107 183 183 183 F F F 28 560-579 104 104 104 180 180 180 F F F 29 580-599 104 104 104 180 180 180 F F F Max gen: 107, Max flush: 183 Begin loop 54 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:16.510952008Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1568,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:16.519043724Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1568,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:16.519324403Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1568} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:16.519352709Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1568} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:16.519361498Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1568} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:16.519372926Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1568} {"msg":"Crucible cf125b54-9c72-4ec6-b7db-dcd9732016c9 has session id: 1aef0876-6b94-4a30-89a0-fbd75c559361","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:16.519665763Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1568,"session_id":"1aef0876-6b94-4a30-89a0-fbd75c559361"} {"msg":"Upstairs opts: Upstairs UUID: cf125b54-9c72-4ec6-b7db-dcd9732016c9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:16.519692279Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1568,"session_id":"1aef0876-6b94-4a30-89a0-fbd75c559361"} {"msg":"Crucible stats registered with UUID: cf125b54-9c72-4ec6-b7db-dcd9732016c9","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:16.519703147Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1568,"session_id":"1aef0876-6b94-4a30-89a0-fbd75c559361"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:57:16.519712396Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1568,"session_id":"1aef0876-6b94-4a30-89a0-fbd75c559361"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:16.519853955Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1568,"":"io task","client":"1","":"downstairs","session_id":"1aef0876-6b94-4a30-89a0-fbd75c559361"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:16.51988526Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1568,"":"io task","client":"0","":"downstairs","session_id":"1aef0876-6b94-4a30-89a0-fbd75c559361"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:16.519896589Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1568,"":"io task","client":"2","":"downstairs","session_id":"1aef0876-6b94-4a30-89a0-fbd75c559361"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:16.520011472Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1568,"task":"crutest"} {"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:16.52002525Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1568} {"msg":"cf125b54-9c72-4ec6-b7db-dcd9732016c9 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:16.520051246Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1568,"session_id":"1aef0876-6b94-4a30-89a0-fbd75c559361"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:16.520060644Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1568,"client":"0","":"downstairs","session_id":"1aef0876-6b94-4a30-89a0-fbd75c559361"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:16.520070363Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1568,"client":"1","":"downstairs","session_id":"1aef0876-6b94-4a30-89a0-fbd75c559361"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:16.520080041Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1568,"client":"2","":"downstairs","session_id":"1aef0876-6b94-4a30-89a0-fbd75c559361"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:16.52008966Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1568,"":"io task","client":"2","":"downstairs","session_id":"1aef0876-6b94-4a30-89a0-fbd75c559361"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:16.520099959Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1568,"":"io task","client":"0","":"downstairs","session_id":"1aef0876-6b94-4a30-89a0-fbd75c559361"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:16.520110117Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1568,"":"io task","client":"1","":"downstairs","session_id":"1aef0876-6b94-4a30-89a0-fbd75c559361"} {"msg":"ds_connection connected from Ok(127.0.0.1:36724)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:16.520207323Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1568,"":"io task","client":"2","":"downstairs","session_id":"1aef0876-6b94-4a30-89a0-fbd75c559361"} {"msg":"ds_connection connected from Ok(127.0.0.1:64623)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:16.520219801Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1568,"":"io task","client":"0","":"downstairs","session_id":"1aef0876-6b94-4a30-89a0-fbd75c559361"} {"msg":"ds_connection connected from Ok(127.0.0.1:36466)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:16.520335464Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1568,"":"io task","client":"1","":"downstairs","session_id":"1aef0876-6b94-4a30-89a0-fbd75c559361"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:16.520872385Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1568,"client":"0","":"downstairs","session_id":"1aef0876-6b94-4a30-89a0-fbd75c559361"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:16.520885763Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1568,"client":"1","":"downstairs","session_id":"1aef0876-6b94-4a30-89a0-fbd75c559361"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:16.520896751Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1568,"client":"2","":"downstairs","session_id":"1aef0876-6b94-4a30-89a0-fbd75c559361"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:16.521224963Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1568,"session_id":"1aef0876-6b94-4a30-89a0-fbd75c559361"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:16.521234771Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1568,"session_id":"1aef0876-6b94-4a30-89a0-fbd75c559361"} {"msg":"[0]R flush_numbers[0..12]: [176, 176, 183, 171, 176, 173, 181, 183, 173, 183, 183, 180]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:16.521271946Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1568,"":"downstairs","":"downstairs","session_id":"1aef0876-6b94-4a30-89a0-fbd75c559361"} {"msg":"[0]R generation[0..12]: [102, 102, 107, 98, 102, 100, 105, 107, 100, 107, 107, 104]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:16.521283134Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1568,"":"downstairs","":"downstairs","session_id":"1aef0876-6b94-4a30-89a0-fbd75c559361"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:16.521294432Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1568,"":"downstairs","":"downstairs","session_id":"1aef0876-6b94-4a30-89a0-fbd75c559361"} {"msg":"[1]R flush_numbers[0..12]: [176, 176, 183, 171, 176, 173, 181, 183, 173, 183, 183, 180]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:16.521304911Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1568,"":"downstairs","":"downstairs","session_id":"1aef0876-6b94-4a30-89a0-fbd75c559361"} {"msg":"[1]R generation[0..12]: [102, 102, 107, 98, 102, 100, 105, 107, 100, 107, 107, 104]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:16.521315419Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1568,"":"downstairs","":"downstairs","session_id":"1aef0876-6b94-4a30-89a0-fbd75c559361"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:16.521326228Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1568,"":"downstairs","":"downstairs","session_id":"1aef0876-6b94-4a30-89a0-fbd75c559361"} {"msg":"[2]R flush_numbers[0..12]: [176, 176, 183, 171, 176, 173, 181, 183, 173, 183, 183, 180]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:16.521336996Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1568,"":"downstairs","":"downstairs","session_id":"1aef0876-6b94-4a30-89a0-fbd75c559361"} {"msg":"[2]R generation[0..12]: [102, 102, 107, 98, 102, 100, 105, 107, 100, 107, 107, 104]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:16.521347614Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1568,"":"downstairs","":"downstairs","session_id":"1aef0876-6b94-4a30-89a0-fbd75c559361"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:16.521358763Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1568,"":"downstairs","":"downstairs","session_id":"1aef0876-6b94-4a30-89a0-fbd75c559361"} {"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:16.521368561Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1568,"":"downstairs","":"downstairs","session_id":"1aef0876-6b94-4a30-89a0-fbd75c559361"} {"msg":"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:16.52137817Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1568,"":"downstairs","":"downstairs","session_id":"1aef0876-6b94-4a30-89a0-fbd75c559361"} {"msg":"Next flush: 184","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:16.521387979Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1568,"":"downstairs","":"downstairs","session_id":"1aef0876-6b94-4a30-89a0-fbd75c559361"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:16.521397437Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1568,"":"downstairs","":"downstairs","session_id":"1aef0876-6b94-4a30-89a0-fbd75c559361"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:16.521409175Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1568,"session_id":"1aef0876-6b94-4a30-89a0-fbd75c559361"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:16.521417844Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1568,"client":"0","":"downstairs","session_id":"1aef0876-6b94-4a30-89a0-fbd75c559361"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:16.521427633Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1568,"client":"1","":"downstairs","session_id":"1aef0876-6b94-4a30-89a0-fbd75c559361"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:16.521437301Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1568,"client":"2","":"downstairs","session_id":"1aef0876-6b94-4a30-89a0-fbd75c559361"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:16.52144715Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1568,"session_id":"1aef0876-6b94-4a30-89a0-fbd75c559361"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:16.521456368Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1568,"session_id":"1aef0876-6b94-4a30-89a0-fbd75c559361"} {"msg":"cf125b54-9c72-4ec6-b7db-dcd9732016c9 is now active with session: 1aef0876-6b94-4a30-89a0-fbd75c559361","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:16.521466397Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1568,"session_id":"1aef0876-6b94-4a30-89a0-fbd75c559361"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:16.521475036Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1568,"session_id":"1aef0876-6b94-4a30-89a0-fbd75c559361"} {"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:16.521484084Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1568} {"msg":"Activated sub_volume cf125b54-9c72-4ec6-b7db-dcd9732016c9","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:16.521491983Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1568} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:16.521500032Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1568,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:16.521800578Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1568,"task":"crutest"} 01/30 Write block 448 len 40960 data: 10 12 12 13 12 12 9 6 9 11 02/30 Read block 123 len 8192 03/30 Read block 327 len 32768 04/30 Read block 564 len 24576 05/30 Read block 454 len 8192 06/30 Read block 551 len 32768 07/30 Read block 126 len 24576 08/30 Read block 387 len 36864 09/30 Flush 10/30 Write block 423 len 24576 data: 9 10 10 7 10 9 11/30 Read block 118 len 28672 12/30 Read block 277 len 40960 13/30 Read block 416 len 12288 14/30 Read block 579 len 16384 15/30 Read block 342 len 20480 16/30 Read block 215 len 4096 17/30 Read block 415 len 4096 18/30 Write block 212 len 28672 data: 9 9 9 9 7 5 4 19/30 Write block 206 len 16384 data: 12 17 16 14 20/30 Write block 448 len 4096 data: 11 21/30 Read block 10 len 32768 22/30 Read block 461 len 36864 23/30 Read block 496 len 20480 24/30 Flush 25/30 Write block 427 len 20480 data: 11 10 9 8 6 26/30 Read block 320 len 40960 27/30 Read block 403 len 28672 28/30 Read block 241 len 16384 29/30 Read block 0 len 12288 30/30 Write block 310 len 32768 data: 12 15 13 13 10 11 11 8 ---------------------------------------------------------------- Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Read 2 Done Done Done false 1002 Acked Read 8 Done Sent Done false 1003 Acked Read 6 Done Sent Done false 1004 Acked Read 2 Done Sent Done false 1005 Acked Read 8 Done Sent Done false 1006 Acked Read 6 Done Sent Done false 1007 Acked Read 9 Done Sent Done false 1008 Acked Flush 0 Done Sent Done false 1009 Acked Write 6 Done Sent Done false 1010 Acked Read 7 Done Sent Done false 1011 Acked Read 10 Done Sent Done false 1012 Acked Read 3 Done Sent Done false 1013 Acked Read 4 Done Sent Done false 1014 Acked Read 5 Done Sent Done false 1015 Acked Read 1 Done Sent Done false 1016 Acked Read 1 Done Sent Done false 1017 Acked Write 7 Done Sent Done false 1018 Acked Write 4 Done Sent Done false 1019 Acked Write 1 Done Sent Done false 1020 Acked Read 8 Done Sent Done false 1021 Acked Read 9 Done Sent Done false 1022 Acked Read 5 Done Sent Done false 1023 Acked Flush 0 Done Sent Done false 1024 Acked Write 5 Done Sent Done false 1025 Acked Read 10 Done Sent Done false 1026 Acked Read 7 Done Sent Done false 1027 Acked Read 4 Done Sent Done false 1028 Acked Read 3 Done Sent Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 28 1 30 Done 29 2 29 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) None Some(JobId(1023)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:57:16.530726181Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1568} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:16.530894916Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1568,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 176 176 176 F F F 1 020-039 102 102 102 176 176 176 F F F 2 040-059 107 107 107 183 183 183 F F F 3 060-079 98 98 98 171 171 171 F F F 4 080-099 102 102 102 176 176 176 F F F 5 100-119 100 100 100 173 173 173 F F F 6 120-139 105 105 105 181 181 181 F F F 7 140-159 107 107 107 183 183 183 F F F 8 160-179 100 100 100 173 173 173 F F F 9 180-199 107 107 107 183 183 183 F F F 10 200-219 108 107 108 185 183 185 F F F <--- 11 220-239 104 104 104 180 180 180 F F F 12 240-259 107 107 107 183 183 183 F F F 13 260-279 105 105 105 181 181 181 F F F 14 280-299 105 105 105 181 181 181 F F F 15 300-319 90 90 90 152 152 152 T F T <--- 16 320-339 106 106 106 182 182 182 F F F 17 340-359 94 94 94 161 161 161 F F F 18 360-379 104 104 104 180 180 180 F F F 19 380-399 107 107 107 183 183 183 F F F 20 400-419 107 107 107 183 183 183 F F F 21 420-439 108 107 108 185 183 185 T F T <--- 22 440-459 108 98 108 185 169 185 F T F <--- 23 460-479 106 106 106 182 182 182 F F F 24 480-499 106 106 106 182 182 182 F F F 25 500-519 107 107 107 183 183 183 F F F 26 520-539 95 95 95 163 163 163 F F F 27 540-559 107 107 107 183 183 183 F F F 28 560-579 104 104 104 180 180 180 F F F 29 580-599 104 104 104 180 180 180 F F F Max gen: 108, Max flush: 185 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 54 Verifying data now No change after verify Loop: 54 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 176 176 176 F F F 1 020-039 102 102 102 176 176 176 F F F 2 040-059 107 107 107 183 183 183 F F F 3 060-079 98 98 98 171 171 171 F F F 4 080-099 102 102 102 176 176 176 F F F 5 100-119 100 100 100 173 173 173 F F F 6 120-139 105 105 105 181 181 181 F F F 7 140-159 107 107 107 183 183 183 F F F 8 160-179 100 100 100 173 173 173 F F F 9 180-199 107 107 107 183 183 183 F F F 10 200-219 108 108 108 185 185 185 F F F 11 220-239 104 104 104 180 180 180 F F F 12 240-259 107 107 107 183 183 183 F F F 13 260-279 105 105 105 181 181 181 F F F 14 280-299 105 105 105 181 181 181 F F F 15 300-319 109 109 109 186 186 186 F F F 16 320-339 106 106 106 182 182 182 F F F 17 340-359 94 94 94 161 161 161 F F F 18 360-379 104 104 104 180 180 180 F F F 19 380-399 107 107 107 183 183 183 F F F 20 400-419 107 107 107 183 183 183 F F F 21 420-439 109 109 109 186 186 186 F F F 22 440-459 108 108 108 185 185 185 F F F 23 460-479 106 106 106 182 182 182 F F F 24 480-499 106 106 106 182 182 182 F F F 25 500-519 107 107 107 183 183 183 F F F 26 520-539 95 95 95 163 163 163 F F F 27 540-559 107 107 107 183 183 183 F F F 28 560-579 104 104 104 180 180 180 F F F 29 580-599 104 104 104 180 180 180 F F F Max gen: 109, Max flush: 186 Begin loop 55 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:20.758445752Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1576,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:20.766497684Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1576,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:20.766800939Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1576} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:20.766839373Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1576} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:20.766851491Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1576} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:20.76686354Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1576} {"msg":"Crucible c22bbdc7-6734-4e7f-98c5-3428c7e42795 has session id: 6f826a75-8feb-42ca-8af7-a4d3b381fcad","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:20.767179993Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1576,"session_id":"6f826a75-8feb-42ca-8af7-a4d3b381fcad"} {"msg":"Upstairs opts: Upstairs UUID: c22bbdc7-6734-4e7f-98c5-3428c7e42795, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:20.76719934Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1576,"session_id":"6f826a75-8feb-42ca-8af7-a4d3b381fcad"} {"msg":"Crucible stats registered with UUID: c22bbdc7-6734-4e7f-98c5-3428c7e42795","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:20.767209399Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1576,"session_id":"6f826a75-8feb-42ca-8af7-a4d3b381fcad"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:57:20.767218507Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1576,"session_id":"6f826a75-8feb-42ca-8af7-a4d3b381fcad"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:20.767364106Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1576,"":"io task","client":"0","":"downstairs","session_id":"6f826a75-8feb-42ca-8af7-a4d3b381fcad"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:20.767381023Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1576,"":"io task","client":"1","":"downstairs","session_id":"6f826a75-8feb-42ca-8af7-a4d3b381fcad"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:20.767391212Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1576,"":"io task","client":"2","":"downstairs","session_id":"6f826a75-8feb-42ca-8af7-a4d3b381fcad"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:20.76754103Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1576,"task":"crutest"} {"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:20.767555748Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1576} {"msg":"c22bbdc7-6734-4e7f-98c5-3428c7e42795 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:20.767579414Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1576,"session_id":"6f826a75-8feb-42ca-8af7-a4d3b381fcad"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:20.767588823Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1576,"client":"0","":"downstairs","session_id":"6f826a75-8feb-42ca-8af7-a4d3b381fcad"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:20.767598631Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1576,"client":"1","":"downstairs","session_id":"6f826a75-8feb-42ca-8af7-a4d3b381fcad"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:20.76760822Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1576,"client":"2","":"downstairs","session_id":"6f826a75-8feb-42ca-8af7-a4d3b381fcad"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:20.767619558Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1576,"":"io task","client":"2","":"downstairs","session_id":"6f826a75-8feb-42ca-8af7-a4d3b381fcad"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:20.767629567Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1576,"":"io task","client":"0","":"downstairs","session_id":"6f826a75-8feb-42ca-8af7-a4d3b381fcad"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:20.767639365Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1576,"":"io task","client":"1","":"downstairs","session_id":"6f826a75-8feb-42ca-8af7-a4d3b381fcad"} {"msg":"ds_connection connected from Ok(127.0.0.1:38841)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:20.767700806Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1576,"":"io task","client":"0","":"downstairs","session_id":"6f826a75-8feb-42ca-8af7-a4d3b381fcad"} {"msg":"ds_connection connected from Ok(127.0.0.1:42254)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:20.767792513Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1576,"":"io task","client":"1","":"downstairs","session_id":"6f826a75-8feb-42ca-8af7-a4d3b381fcad"} {"msg":"ds_connection connected from Ok(127.0.0.1:36134)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:20.76781378Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1576,"":"io task","client":"2","":"downstairs","session_id":"6f826a75-8feb-42ca-8af7-a4d3b381fcad"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:20.768405782Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1576,"client":"2","":"downstairs","session_id":"6f826a75-8feb-42ca-8af7-a4d3b381fcad"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:20.768432228Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1576,"client":"0","":"downstairs","session_id":"6f826a75-8feb-42ca-8af7-a4d3b381fcad"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:20.768444037Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1576,"client":"1","":"downstairs","session_id":"6f826a75-8feb-42ca-8af7-a4d3b381fcad"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:20.768713187Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1576,"session_id":"6f826a75-8feb-42ca-8af7-a4d3b381fcad"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:20.768722795Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1576,"session_id":"6f826a75-8feb-42ca-8af7-a4d3b381fcad"} {"msg":"[0]R flush_numbers[0..12]: [176, 176, 183, 171, 176, 173, 181, 183, 173, 183, 185, 180]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:20.768819141Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1576,"":"downstairs","":"downstairs","session_id":"6f826a75-8feb-42ca-8af7-a4d3b381fcad"} {"msg":"[0]R generation[0..12]: [102, 102, 107, 98, 102, 100, 105, 107, 100, 107, 108, 104]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:20.76883023Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1576,"":"downstairs","":"downstairs","session_id":"6f826a75-8feb-42ca-8af7-a4d3b381fcad"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:20.768841398Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1576,"":"downstairs","":"downstairs","session_id":"6f826a75-8feb-42ca-8af7-a4d3b381fcad"} {"msg":"[1]R flush_numbers[0..12]: [176, 176, 183, 171, 176, 173, 181, 183, 173, 183, 185, 180]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:20.768857426Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1576,"":"downstairs","":"downstairs","session_id":"6f826a75-8feb-42ca-8af7-a4d3b381fcad"} {"msg":"[1]R generation[0..12]: [102, 102, 107, 98, 102, 100, 105, 107, 100, 107, 108, 104]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:20.768870584Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1576,"":"downstairs","":"downstairs","session_id":"6f826a75-8feb-42ca-8af7-a4d3b381fcad"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:20.768881382Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1576,"":"downstairs","":"downstairs","session_id":"6f826a75-8feb-42ca-8af7-a4d3b381fcad"} {"msg":"[2]R flush_numbers[0..12]: [176, 176, 183, 171, 176, 173, 181, 183, 173, 183, 185, 180]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:20.76889213Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1576,"":"downstairs","":"downstairs","session_id":"6f826a75-8feb-42ca-8af7-a4d3b381fcad"} {"msg":"[2]R generation[0..12]: [102, 102, 107, 98, 102, 100, 105, 107, 100, 107, 108, 104]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:20.768902679Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1576,"":"downstairs","":"downstairs","session_id":"6f826a75-8feb-42ca-8af7-a4d3b381fcad"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:20.768913507Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1576,"":"downstairs","":"downstairs","session_id":"6f826a75-8feb-42ca-8af7-a4d3b381fcad"} {"msg":"Max found gen is 110","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:20.768922876Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1576,"":"downstairs","":"downstairs","session_id":"6f826a75-8feb-42ca-8af7-a4d3b381fcad"} {"msg":"Generation requested: 110 >= found:110","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:20.768932684Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1576,"":"downstairs","":"downstairs","session_id":"6f826a75-8feb-42ca-8af7-a4d3b381fcad"} {"msg":"Next flush: 187","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:20.768941923Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1576,"":"downstairs","":"downstairs","session_id":"6f826a75-8feb-42ca-8af7-a4d3b381fcad"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:20.768951352Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1576,"":"downstairs","":"downstairs","session_id":"6f826a75-8feb-42ca-8af7-a4d3b381fcad"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:20.76896111Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1576,"session_id":"6f826a75-8feb-42ca-8af7-a4d3b381fcad"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:20.768969989Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1576,"client":"0","":"downstairs","session_id":"6f826a75-8feb-42ca-8af7-a4d3b381fcad"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:20.768979688Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1576,"client":"1","":"downstairs","session_id":"6f826a75-8feb-42ca-8af7-a4d3b381fcad"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:20.768989336Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1576,"client":"2","":"downstairs","session_id":"6f826a75-8feb-42ca-8af7-a4d3b381fcad"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:20.768999245Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1576,"session_id":"6f826a75-8feb-42ca-8af7-a4d3b381fcad"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:20.769008303Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1576,"session_id":"6f826a75-8feb-42ca-8af7-a4d3b381fcad"} {"msg":"c22bbdc7-6734-4e7f-98c5-3428c7e42795 is now active with session: 6f826a75-8feb-42ca-8af7-a4d3b381fcad","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:20.769019472Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1576,"session_id":"6f826a75-8feb-42ca-8af7-a4d3b381fcad"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:20.7690282Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1576,"session_id":"6f826a75-8feb-42ca-8af7-a4d3b381fcad"} {"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:20.769037569Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1576} {"msg":"Activated sub_volume c22bbdc7-6734-4e7f-98c5-3428c7e42795","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:20.769045508Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1576} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:20.769053547Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1576,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:20.769396526Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1576,"task":"crutest"} 01/30 Write block 165 len 24576 data: 7 7 7 5 5 4 02/30 Read block 492 len 16384 03/30 Write block 465 len 24576 data: 9 13 11 12 12 10 04/30 Read block 142 len 40960 05/30 Read block 482 len 36864 06/30 Write block 257 len 32768 data: 13 14 12 7 7 6 10 10 07/30 Write block 419 len 36864 data: 6 6 6 9 10 11 11 8 12 08/30 Flush 09/30 Flush 10/30 Write block 231 len 12288 data: 7 9 7 11/30 Write block 185 len 4096 data: 6 12/30 Write block 39 len 4096 data: 15 13/30 Write block 118 len 20480 data: 7 8 8 10 8 14/30 Read block 50 len 12288 15/30 Read block 446 len 20480 16/30 Read block 212 len 12288 17/30 Write block 380 len 16384 data: 6 7 10 9 18/30 Read block 425 len 16384 19/30 Flush 20/30 Flush 21/30 Write block 8 len 36864 data: 7 5 5 4 6 6 6 10 10 22/30 Flush 23/30 Write block 494 len 28672 data: 5 7 7 6 5 7 8 24/30 Flush 25/30 Write block 362 len 4096 data: 5 26/30 Write block 142 len 16384 data: 11 10 13 12 27/30 Read block 16 len 16384 28/30 Write block 487 len 12288 data: 8 11 8 29/30 Read block 96 len 8192 30/30 Write block 173 len 36864 data: 6 6 7 9 10 10 8 7 7 ---------------------------------------------------------------- Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Sent Done false 1001 Acked Read 4 Done Sent Done false 1002 Acked Write 6 Done Sent Done false 1003 Acked Read 10 Done Sent Done false 1004 Acked Read 9 Done Sent Done false 1005 Acked Write 8 Done Sent Done false 1006 Acked Write 9 Done Sent Done false 1007 Acked Flush 0 Done Sent Done false 1008 Acked Flush 0 Done Sent Done false 1009 Acked Write 3 Done Sent Done false 1010 Acked Write 1 Done Sent Done false 1011 Acked Write 1 Done Sent Done false 1012 Acked Write 5 Done Sent Done false 1013 Acked Read 3 Done Sent Done false 1014 Acked Read 5 Done Sent Done false 1015 Acked Read 3 Done Sent Done false 1016 Acked Write 4 Done Sent Done false 1017 Acked Read 4 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Flush 0 Done Sent Done false 1020 Acked Write 9 Done Sent Done false 1021 Acked Flush 0 Done Sent Done false 1022 Acked Write 7 Done Sent Done false 1023 Acked Flush 0 Done Sent Done false 1024 Acked Write 1 Done Sent Done false 1025 Acked Write 4 Done Sent Done false 1026 Acked Read 4 Done Sent Done false 1027 Acked Write 3 Done Sent Done false 1028 Acked Read 2 Done Sent Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 30 1 32 Done 29 0 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) None Some(JobId(1023)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:57:20.778275676Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1576} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:20.778436212Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1576,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 110 102 110 191 176 191 F F F <--- 1 020-039 110 102 110 189 176 189 F F F <--- 2 040-059 107 107 107 183 183 183 F F F 3 060-079 98 98 98 171 171 171 F F F 4 080-099 102 102 102 176 176 176 F F F 5 100-119 110 100 110 189 173 189 F F F <--- 6 120-139 110 105 110 189 181 189 F F F <--- 7 140-159 107 107 107 183 183 183 T F T <--- 8 160-179 110 100 110 187 173 187 T F T <--- 9 180-199 110 107 110 189 183 189 T F T <--- 10 200-219 108 108 108 185 185 185 F F F 11 220-239 110 104 110 189 180 189 F F F <--- 12 240-259 110 107 110 187 183 187 F F F <--- 13 260-279 110 105 110 187 181 187 F F F <--- 14 280-299 105 105 105 181 181 181 F F F 15 300-319 109 109 109 186 186 186 F F F 16 320-339 106 106 106 182 182 182 F F F 17 340-359 94 94 94 161 161 161 F F F 18 360-379 104 104 104 180 180 180 T F T <--- 19 380-399 110 107 110 189 183 189 F F F <--- 20 400-419 110 107 110 187 183 187 F F F <--- 21 420-439 110 109 110 187 186 187 F F F <--- 22 440-459 108 108 108 185 185 185 F F F 23 460-479 110 106 110 187 182 187 F F F <--- 24 480-499 110 106 110 192 182 192 T F T <--- 25 500-519 110 107 110 192 183 192 F F F <--- 26 520-539 95 95 95 163 163 163 F F F 27 540-559 107 107 107 183 183 183 F F F 28 560-579 104 104 104 180 180 180 F F F 29 580-599 104 104 104 180 180 180 F F F Max gen: 110, Max flush: 192 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 55 Verifying data now No change after verify Loop: 55 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 110 110 110 191 191 191 F F F 1 020-039 110 110 110 189 189 189 F F F 2 040-059 107 107 107 183 183 183 F F F 3 060-079 98 98 98 171 171 171 F F F 4 080-099 102 102 102 176 176 176 F F F 5 100-119 110 110 110 189 189 189 F F F 6 120-139 110 110 110 189 189 189 F F F 7 140-159 111 111 111 193 193 193 F F F 8 160-179 111 111 111 193 193 193 F F F 9 180-199 111 111 111 193 193 193 F F F 10 200-219 108 108 108 185 185 185 F F F 11 220-239 110 110 110 189 189 189 F F F 12 240-259 110 110 110 187 187 187 F F F 13 260-279 110 110 110 187 187 187 F F F 14 280-299 105 105 105 181 181 181 F F F 15 300-319 109 109 109 186 186 186 F F F 16 320-339 106 106 106 182 182 182 F F F 17 340-359 94 94 94 161 161 161 F F F 18 360-379 111 111 111 193 193 193 F F F 19 380-399 110 110 110 189 189 189 F F F 20 400-419 110 110 110 187 187 187 F F F 21 420-439 110 110 110 187 187 187 F F F 22 440-459 108 108 108 185 185 185 F F F 23 460-479 110 110 110 187 187 187 F F F 24 480-499 111 111 111 193 193 193 F F F 25 500-519 110 110 110 192 192 192 F F F 26 520-539 95 95 95 163 163 163 F F F 27 540-559 107 107 107 183 183 183 F F F 28 560-579 104 104 104 180 180 180 F F F 29 580-599 104 104 104 180 180 180 F F F Max gen: 111, Max flush: 193 Begin loop 56 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:25.036279748Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1584,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:57:25.043387089Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1584,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:25.043435912Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1584,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:25.050330545Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1584,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:25.050598596Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1584} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:25.050627751Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1584} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:25.05063625Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1584} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:25.050648008Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1584} {"msg":"Crucible f804fecd-5987-4515-9376-42f27b390e62 has session id: aef3bd38-97e0-40f6-a133-7eba0cba4dc7","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:25.050931226Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1584,"session_id":"aef3bd38-97e0-40f6-a133-7eba0cba4dc7"} {"msg":"Upstairs opts: Upstairs UUID: f804fecd-5987-4515-9376-42f27b390e62, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:25.050964082Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1584,"session_id":"aef3bd38-97e0-40f6-a133-7eba0cba4dc7"} {"msg":"Crucible stats registered with UUID: f804fecd-5987-4515-9376-42f27b390e62","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:25.05097541Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1584,"session_id":"aef3bd38-97e0-40f6-a133-7eba0cba4dc7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:57:25.050984949Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1584,"session_id":"aef3bd38-97e0-40f6-a133-7eba0cba4dc7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:25.05111113Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1584,"":"io task","client":"1","":"downstairs","session_id":"aef3bd38-97e0-40f6-a133-7eba0cba4dc7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:25.051127857Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1584,"":"io task","client":"0","":"downstairs","session_id":"aef3bd38-97e0-40f6-a133-7eba0cba4dc7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:25.051138316Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1584,"":"io task","client":"2","":"downstairs","session_id":"aef3bd38-97e0-40f6-a133-7eba0cba4dc7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:25.051272376Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1584,"task":"crutest"} {"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:25.051287824Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1584} {"msg":"f804fecd-5987-4515-9376-42f27b390e62 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:25.051326658Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1584,"session_id":"aef3bd38-97e0-40f6-a133-7eba0cba4dc7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:25.051336337Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1584,"client":"0","":"downstairs","session_id":"aef3bd38-97e0-40f6-a133-7eba0cba4dc7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:25.051346555Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1584,"client":"1","":"downstairs","session_id":"aef3bd38-97e0-40f6-a133-7eba0cba4dc7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:25.051356474Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1584,"client":"2","":"downstairs","session_id":"aef3bd38-97e0-40f6-a133-7eba0cba4dc7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:25.051366342Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1584,"":"io task","client":"0","":"downstairs","session_id":"aef3bd38-97e0-40f6-a133-7eba0cba4dc7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:25.051376681Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1584,"":"io task","client":"2","":"downstairs","session_id":"aef3bd38-97e0-40f6-a133-7eba0cba4dc7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:25.051386659Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1584,"":"io task","client":"1","":"downstairs","session_id":"aef3bd38-97e0-40f6-a133-7eba0cba4dc7"} {"msg":"ds_connection connected from Ok(127.0.0.1:47287)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:25.051436372Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1584,"":"io task","client":"0","":"downstairs","session_id":"aef3bd38-97e0-40f6-a133-7eba0cba4dc7"} {"msg":"ds_connection connected from Ok(127.0.0.1:55293)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:25.051550855Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1584,"":"io task","client":"2","":"downstairs","session_id":"aef3bd38-97e0-40f6-a133-7eba0cba4dc7"} {"msg":"ds_connection connected from Ok(127.0.0.1:43326)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:25.051577241Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1584,"":"io task","client":"1","":"downstairs","session_id":"aef3bd38-97e0-40f6-a133-7eba0cba4dc7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:25.052171323Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1584,"client":"1","":"downstairs","session_id":"aef3bd38-97e0-40f6-a133-7eba0cba4dc7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:25.05219289Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1584,"client":"0","":"downstairs","session_id":"aef3bd38-97e0-40f6-a133-7eba0cba4dc7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:25.052204549Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1584,"client":"2","":"downstairs","session_id":"aef3bd38-97e0-40f6-a133-7eba0cba4dc7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:25.052473619Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1584,"session_id":"aef3bd38-97e0-40f6-a133-7eba0cba4dc7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:25.052483367Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1584,"session_id":"aef3bd38-97e0-40f6-a133-7eba0cba4dc7"} {"msg":"[0]R flush_numbers[0..12]: [191, 189, 183, 171, 176, 189, 189, 193, 193, 193, 185, 189]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:25.052516443Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1584,"":"downstairs","":"downstairs","session_id":"aef3bd38-97e0-40f6-a133-7eba0cba4dc7"} {"msg":"[0]R generation[0..12]: [110, 110, 107, 98, 102, 110, 110, 111, 111, 111, 108, 110]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:25.052527671Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1584,"":"downstairs","":"downstairs","session_id":"aef3bd38-97e0-40f6-a133-7eba0cba4dc7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:25.052538749Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1584,"":"downstairs","":"downstairs","session_id":"aef3bd38-97e0-40f6-a133-7eba0cba4dc7"} {"msg":"[1]R flush_numbers[0..12]: [191, 189, 183, 171, 176, 189, 189, 193, 193, 193, 185, 189]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:25.052549458Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1584,"":"downstairs","":"downstairs","session_id":"aef3bd38-97e0-40f6-a133-7eba0cba4dc7"} {"msg":"[1]R generation[0..12]: [110, 110, 107, 98, 102, 110, 110, 111, 111, 111, 108, 110]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:25.052560096Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1584,"":"downstairs","":"downstairs","session_id":"aef3bd38-97e0-40f6-a133-7eba0cba4dc7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:25.052570865Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1584,"":"downstairs","":"downstairs","session_id":"aef3bd38-97e0-40f6-a133-7eba0cba4dc7"} {"msg":"[2]R flush_numbers[0..12]: [191, 189, 183, 171, 176, 189, 189, 193, 193, 193, 185, 189]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:25.052581673Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1584,"":"downstairs","":"downstairs","session_id":"aef3bd38-97e0-40f6-a133-7eba0cba4dc7"} {"msg":"[2]R generation[0..12]: [110, 110, 107, 98, 102, 110, 110, 111, 111, 111, 108, 110]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:25.052594821Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1584,"":"downstairs","":"downstairs","session_id":"aef3bd38-97e0-40f6-a133-7eba0cba4dc7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:25.052605949Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1584,"":"downstairs","":"downstairs","session_id":"aef3bd38-97e0-40f6-a133-7eba0cba4dc7"} {"msg":"Max found gen is 112","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:25.052615418Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1584,"":"downstairs","":"downstairs","session_id":"aef3bd38-97e0-40f6-a133-7eba0cba4dc7"} {"msg":"Generation requested: 112 >= found:112","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:25.052625207Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1584,"":"downstairs","":"downstairs","session_id":"aef3bd38-97e0-40f6-a133-7eba0cba4dc7"} {"msg":"Next flush: 194","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:25.052634625Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1584,"":"downstairs","":"downstairs","session_id":"aef3bd38-97e0-40f6-a133-7eba0cba4dc7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:25.052644004Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1584,"":"downstairs","":"downstairs","session_id":"aef3bd38-97e0-40f6-a133-7eba0cba4dc7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:25.052653872Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1584,"session_id":"aef3bd38-97e0-40f6-a133-7eba0cba4dc7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:25.052662641Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1584,"client":"0","":"downstairs","session_id":"aef3bd38-97e0-40f6-a133-7eba0cba4dc7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:25.05267231Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1584,"client":"1","":"downstairs","session_id":"aef3bd38-97e0-40f6-a133-7eba0cba4dc7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:25.052682078Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1584,"client":"2","":"downstairs","session_id":"aef3bd38-97e0-40f6-a133-7eba0cba4dc7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:25.052692227Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1584,"session_id":"aef3bd38-97e0-40f6-a133-7eba0cba4dc7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:25.052701485Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1584,"session_id":"aef3bd38-97e0-40f6-a133-7eba0cba4dc7"} {"msg":"f804fecd-5987-4515-9376-42f27b390e62 is now active with session: aef3bd38-97e0-40f6-a133-7eba0cba4dc7","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:25.052711484Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1584,"session_id":"aef3bd38-97e0-40f6-a133-7eba0cba4dc7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:25.052720392Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1584,"session_id":"aef3bd38-97e0-40f6-a133-7eba0cba4dc7"} {"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:25.052729521Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1584} {"msg":"Activated sub_volume f804fecd-5987-4515-9376-42f27b390e62","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:25.05273766Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1584} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:25.052752738Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1584,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:25.05300501Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1584,"task":"crutest"} 01/30 Write block 594 len 20480 data: 12 11 9 6 4 02/30 Read block 462 len 40960 03/30 Write block 257 len 24576 data: 14 15 13 8 8 7 04/30 Read block 261 len 16384 05/30 Read block 521 len 24576 06/30 Read block 200 len 4096 07/30 Write block 323 len 12288 data: 11 12 13 08/30 Read block 358 len 12288 09/30 Read block 569 len 4096 10/30 Write block 6 len 16384 data: 8 7 8 6 11/30 Read block 261 len 16384 12/30 Read block 361 len 16384 13/30 Write block 583 len 36864 data: 11 13 13 12 12 8 8 6 8 14/30 Write block 353 len 32768 data: 8 10 9 10 11 11 9 10 15/30 Read block 457 len 40960 16/30 Write block 198 len 32768 data: 10 9 7 5 6 8 10 11 17/30 Write block 579 len 24576 data: 7 7 11 11 12 14 18/30 Read block 336 len 4096 19/30 Write block 554 len 24576 data: 12 10 9 10 9 7 20/30 Read block 342 len 20480 21/30 Write block 124 len 28672 data: 8 7 7 7 4 6 5 22/30 Flush 23/30 Write block 532 len 24576 data: 8 9 9 9 10 7 24/30 Read block 585 len 20480 25/30 Read block 164 len 20480 26/30 Flush 27/30 Write block 505 len 24576 data: 4 4 6 4 4 8 28/30 Write block 530 len 8192 data: 5 7 29/30 Write block 118 len 36864 data: 8 9 9 11 9 10 9 8 8 30/30 Read block 187 len 24576 ---------------------------------------------------------------- Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Read 10 Done Done Done false 1002 Acked Write 6 Done Done Done false 1003 Acked Read 4 Sent Done Done false 1004 Acked Read 6 Sent Done Done false 1005 Acked Read 1 Sent Done Done false 1006 Acked Write 3 Sent Done Done false 1007 Acked Read 3 Sent Done Done false 1008 Acked Read 1 Sent Done Done false 1009 Acked Write 4 Sent Done Done false 1010 Acked Read 4 Sent Done Done false 1011 Acked Read 4 Sent Done Done false 1012 Acked Write 9 Sent Done Done false 1013 Acked Write 8 Sent Done Done false 1014 Acked Read 10 Sent Done Done false 1015 Acked Write 8 Sent Done Done false 1016 Acked Write 6 Sent Done Done false 1017 Acked Read 1 Sent Done Done false 1018 Acked Write 6 Sent Done Done false 1019 Acked Read 5 Sent Done Done false 1020 Acked Write 7 Sent Done Done false 1021 Acked Flush 0 Sent Done Done false 1022 Acked Write 6 Sent Done Done false 1023 Acked Read 5 Sent Done Done false 1024 Acked Read 5 Sent Done Done false 1025 Acked Flush 0 Sent Done Done false 1026 Acked Write 6 Sent Done Done false 1027 Acked Write 2 Sent Done Done false 1028 Acked Write 9 Sent Done Done false 1029 Acked Read 6 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 27 0 1 28 Done 3 30 29 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:57:25.06243018Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1584} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:25.062601455Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1584,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 110 112 112 191 194 194 F F F <--- 1 020-039 110 110 110 189 189 189 F F F 2 040-059 107 107 107 183 183 183 F F F 3 060-079 98 98 98 171 171 171 F F F 4 080-099 102 102 102 176 176 176 F F F 5 100-119 110 110 110 189 189 189 F T T <--- 6 120-139 110 112 112 189 194 194 F T T <--- 7 140-159 111 111 111 193 193 193 F F F 8 160-179 111 111 111 193 193 193 F F F 9 180-199 111 112 112 193 194 194 F F F <--- 10 200-219 108 112 112 185 194 194 F F F <--- 11 220-239 110 110 110 189 189 189 F F F 12 240-259 110 112 112 187 194 194 T F F <--- 13 260-279 110 112 112 187 194 194 T F F <--- 14 280-299 105 105 105 181 181 181 F F F 15 300-319 109 109 109 186 186 186 F F F 16 320-339 106 112 112 182 194 194 F F F <--- 17 340-359 94 112 112 161 194 194 F F F <--- 18 360-379 111 112 112 193 194 194 F F F <--- 19 380-399 110 110 110 189 189 189 F F F 20 400-419 110 110 110 187 187 187 F F F 21 420-439 110 110 110 187 187 187 F F F 22 440-459 108 108 108 185 185 185 F F F 23 460-479 110 110 110 187 187 187 F F F 24 480-499 111 111 111 193 193 193 F F F 25 500-519 110 110 110 192 192 192 F T T <--- 26 520-539 95 112 112 163 195 195 F T T <--- 27 540-559 107 112 112 183 194 194 F F F <--- 28 560-579 104 112 112 180 194 194 F F F <--- 29 580-599 104 112 112 180 194 194 T F F <--- Max gen: 112, Max flush: 195 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 56 Verifying data now No change after verify Loop: 56 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 194 194 194 F F F 1 020-039 110 110 110 189 189 189 F F F 2 040-059 107 107 107 183 183 183 F F F 3 060-079 98 98 98 171 171 171 F F F 4 080-099 102 102 102 176 176 176 F F F 5 100-119 113 113 113 196 196 196 F F F 6 120-139 113 113 113 196 196 196 F F F 7 140-159 111 111 111 193 193 193 F F F 8 160-179 111 111 111 193 193 193 F F F 9 180-199 112 112 112 194 194 194 F F F 10 200-219 112 112 112 194 194 194 F F F 11 220-239 110 110 110 189 189 189 F F F 12 240-259 112 112 112 194 194 194 F F F 13 260-279 112 112 112 194 194 194 F F F 14 280-299 105 105 105 181 181 181 F F F 15 300-319 109 109 109 186 186 186 F F F 16 320-339 112 112 112 194 194 194 F F F 17 340-359 112 112 112 194 194 194 F F F 18 360-379 112 112 112 194 194 194 F F F 19 380-399 110 110 110 189 189 189 F F F 20 400-419 110 110 110 187 187 187 F F F 21 420-439 110 110 110 187 187 187 F F F 22 440-459 108 108 108 185 185 185 F F F 23 460-479 110 110 110 187 187 187 F F F 24 480-499 111 111 111 193 193 193 F F F 25 500-519 113 113 113 196 196 196 F F F 26 520-539 113 113 113 196 196 196 F F F 27 540-559 112 112 112 194 194 194 F F F 28 560-579 112 112 112 194 194 194 F F F 29 580-599 112 112 112 194 194 194 F F F Max gen: 113, Max flush: 196 Begin loop 57 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:29.323214583Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1592,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:29.331537025Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1592,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:29.331831011Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1592} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:29.331855158Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1592} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:29.331864036Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1592} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:29.331875615Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1592} {"msg":"Crucible 748b1921-8ef1-4637-bd1d-afdd0bd287ce has session id: 70dce9ef-a59e-41f4-83b7-9a0936c5366c","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:29.332194028Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1592,"session_id":"70dce9ef-a59e-41f4-83b7-9a0936c5366c"} {"msg":"Upstairs opts: Upstairs UUID: 748b1921-8ef1-4637-bd1d-afdd0bd287ce, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:29.332223243Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1592,"session_id":"70dce9ef-a59e-41f4-83b7-9a0936c5366c"} {"msg":"Crucible stats registered with UUID: 748b1921-8ef1-4637-bd1d-afdd0bd287ce","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:29.332234422Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1592,"session_id":"70dce9ef-a59e-41f4-83b7-9a0936c5366c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:57:29.33224358Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1592,"session_id":"70dce9ef-a59e-41f4-83b7-9a0936c5366c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:29.332364962Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1592,"":"io task","client":"2","":"downstairs","session_id":"70dce9ef-a59e-41f4-83b7-9a0936c5366c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:29.332395108Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1592,"":"io task","client":"0","":"downstairs","session_id":"70dce9ef-a59e-41f4-83b7-9a0936c5366c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:29.332408496Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1592,"":"io task","client":"1","":"downstairs","session_id":"70dce9ef-a59e-41f4-83b7-9a0936c5366c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:29.332536977Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1592,"task":"crutest"} {"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:29.332554864Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1592} {"msg":"748b1921-8ef1-4637-bd1d-afdd0bd287ce active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:29.33258578Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1592,"session_id":"70dce9ef-a59e-41f4-83b7-9a0936c5366c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:29.332595458Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1592,"client":"0","":"downstairs","session_id":"70dce9ef-a59e-41f4-83b7-9a0936c5366c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:29.332605177Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1592,"client":"1","":"downstairs","session_id":"70dce9ef-a59e-41f4-83b7-9a0936c5366c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:29.332614906Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1592,"client":"2","":"downstairs","session_id":"70dce9ef-a59e-41f4-83b7-9a0936c5366c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:29.332624584Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1592,"":"io task","client":"2","":"downstairs","session_id":"70dce9ef-a59e-41f4-83b7-9a0936c5366c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:29.332634613Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1592,"":"io task","client":"0","":"downstairs","session_id":"70dce9ef-a59e-41f4-83b7-9a0936c5366c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:29.332644711Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1592,"":"io task","client":"1","":"downstairs","session_id":"70dce9ef-a59e-41f4-83b7-9a0936c5366c"} {"msg":"ds_connection connected from Ok(127.0.0.1:56563)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:29.33272324Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1592,"":"io task","client":"0","":"downstairs","session_id":"70dce9ef-a59e-41f4-83b7-9a0936c5366c"} {"msg":"ds_connection connected from Ok(127.0.0.1:43934)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:29.332748746Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1592,"":"io task","client":"2","":"downstairs","session_id":"70dce9ef-a59e-41f4-83b7-9a0936c5366c"} {"msg":"ds_connection connected from Ok(127.0.0.1:44518)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:29.332760324Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1592,"":"io task","client":"1","":"downstairs","session_id":"70dce9ef-a59e-41f4-83b7-9a0936c5366c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:29.333454392Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1592,"client":"2","":"downstairs","session_id":"70dce9ef-a59e-41f4-83b7-9a0936c5366c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:29.333472409Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1592,"client":"1","":"downstairs","session_id":"70dce9ef-a59e-41f4-83b7-9a0936c5366c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:29.333483267Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1592,"client":"0","":"downstairs","session_id":"70dce9ef-a59e-41f4-83b7-9a0936c5366c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:29.333777864Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1592,"session_id":"70dce9ef-a59e-41f4-83b7-9a0936c5366c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:29.33380268Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1592,"session_id":"70dce9ef-a59e-41f4-83b7-9a0936c5366c"} {"msg":"[0]R flush_numbers[0..12]: [194, 189, 183, 171, 176, 196, 196, 193, 193, 194, 194, 189]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:29.333863361Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1592,"":"downstairs","":"downstairs","session_id":"70dce9ef-a59e-41f4-83b7-9a0936c5366c"} {"msg":"[0]R generation[0..12]: [112, 110, 107, 98, 102, 113, 113, 111, 111, 112, 112, 110]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:29.33387484Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1592,"":"downstairs","":"downstairs","session_id":"70dce9ef-a59e-41f4-83b7-9a0936c5366c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:29.333885738Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1592,"":"downstairs","":"downstairs","session_id":"70dce9ef-a59e-41f4-83b7-9a0936c5366c"} {"msg":"[1]R flush_numbers[0..12]: [194, 189, 183, 171, 176, 196, 196, 193, 193, 194, 194, 189]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:29.333896427Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1592,"":"downstairs","":"downstairs","session_id":"70dce9ef-a59e-41f4-83b7-9a0936c5366c"} {"msg":"[1]R generation[0..12]: [112, 110, 107, 98, 102, 113, 113, 111, 111, 112, 112, 110]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:29.333906885Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1592,"":"downstairs","":"downstairs","session_id":"70dce9ef-a59e-41f4-83b7-9a0936c5366c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:29.333917843Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1592,"":"downstairs","":"downstairs","session_id":"70dce9ef-a59e-41f4-83b7-9a0936c5366c"} {"msg":"[2]R flush_numbers[0..12]: [194, 189, 183, 171, 176, 196, 196, 193, 193, 194, 194, 189]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:29.333928422Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1592,"":"downstairs","":"downstairs","session_id":"70dce9ef-a59e-41f4-83b7-9a0936c5366c"} {"msg":"[2]R generation[0..12]: [112, 110, 107, 98, 102, 113, 113, 111, 111, 112, 112, 110]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:29.33393892Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1592,"":"downstairs","":"downstairs","session_id":"70dce9ef-a59e-41f4-83b7-9a0936c5366c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:29.333949769Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1592,"":"downstairs","":"downstairs","session_id":"70dce9ef-a59e-41f4-83b7-9a0936c5366c"} {"msg":"Max found gen is 114","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:29.333959487Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1592,"":"downstairs","":"downstairs","session_id":"70dce9ef-a59e-41f4-83b7-9a0936c5366c"} {"msg":"Generation requested: 114 >= found:114","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:29.333969186Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1592,"":"downstairs","":"downstairs","session_id":"70dce9ef-a59e-41f4-83b7-9a0936c5366c"} {"msg":"Next flush: 197","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:29.333978644Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1592,"":"downstairs","":"downstairs","session_id":"70dce9ef-a59e-41f4-83b7-9a0936c5366c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:29.333989683Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1592,"":"downstairs","":"downstairs","session_id":"70dce9ef-a59e-41f4-83b7-9a0936c5366c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:29.333999411Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1592,"session_id":"70dce9ef-a59e-41f4-83b7-9a0936c5366c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:29.33400814Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1592,"client":"0","":"downstairs","session_id":"70dce9ef-a59e-41f4-83b7-9a0936c5366c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:29.334017889Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1592,"client":"1","":"downstairs","session_id":"70dce9ef-a59e-41f4-83b7-9a0936c5366c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:29.334027507Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1592,"client":"2","":"downstairs","session_id":"70dce9ef-a59e-41f4-83b7-9a0936c5366c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:29.334037086Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1592,"session_id":"70dce9ef-a59e-41f4-83b7-9a0936c5366c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:29.334046334Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1592,"session_id":"70dce9ef-a59e-41f4-83b7-9a0936c5366c"} {"msg":"748b1921-8ef1-4637-bd1d-afdd0bd287ce is now active with session: 70dce9ef-a59e-41f4-83b7-9a0936c5366c","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:29.334056293Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1592,"session_id":"70dce9ef-a59e-41f4-83b7-9a0936c5366c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:29.334065172Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1592,"session_id":"70dce9ef-a59e-41f4-83b7-9a0936c5366c"} {"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:29.33407429Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1592} {"msg":"Activated sub_volume 748b1921-8ef1-4637-bd1d-afdd0bd287ce","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:29.334082279Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1592} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:29.334090398Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1592,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:29.334408161Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1592,"task":"crutest"} 01/30 Write block 428 len 32768 data: 11 10 9 7 8 9 9 9 02/30 Read block 452 len 4096 03/30 Read block 358 len 16384 04/30 Write block 408 len 4096 data: 9 05/30 Read block 80 len 8192 06/30 Flush 07/30 Write block 171 len 24576 data: 5 6 7 7 8 10 08/30 Read block 69 len 32768 09/30 Read block 400 len 12288 10/30 Read block 83 len 12288 11/30 Write block 512 len 40960 data: 9 11 11 11 10 9 12 11 10 10 12/30 Write block 463 len 36864 data: 7 9 10 14 12 13 13 11 11 13/30 Read block 433 len 12288 14/30 Write block 496 len 40960 data: 8 7 6 8 9 6 8 7 5 5 15/30 Write block 62 len 28672 data: 7 6 6 5 6 9 7 16/30 Write block 450 len 20480 data: 13 14 13 13 10 17/30 Read block 37 len 16384 18/30 Flush 19/30 Write block 583 len 8192 data: 13 15 20/30 Read block 362 len 28672 21/30 Read block 57 len 8192 22/30 Write block 501 len 24576 data: 7 9 8 6 6 5 23/30 Read block 214 len 32768 24/30 Read block 463 len 32768 25/30 Read block 538 len 16384 26/30 Read block 325 len 40960 27/30 Write block 588 len 4096 data: 9 28/30 Write block 356 len 8192 data: 11 12 29/30 Write block 424 len 8192 data: 12 12 30/30 Write block 297 len 24576 data: 3 3 4 7 8 7 ---------------------------------------------------------------- Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Read 1 Done Done Done false 1002 Acked Read 4 Done Done Done false 1003 Acked Write 1 Done Done Done false 1004 Acked Read 2 Done Done Done false 1005 Acked Flush 0 Done Sent Done false 1006 Acked Write 6 Done Sent Done false 1007 Acked Read 8 Done Sent Done false 1008 Acked Read 3 Done Sent Done false 1009 Acked Read 3 Done Sent Done false 1010 Acked Write 10 Done Sent Done false 1011 Acked Write 9 Done Sent Done false 1012 Acked Read 3 Done Sent Done false 1013 Acked Write 10 Done Sent Done false 1014 Acked Write 7 Done Sent Done false 1015 Acked Write 5 Done Sent Done false 1016 Acked Read 4 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Write 2 Done Sent Done false 1019 Acked Read 7 Done Sent Done false 1020 Acked Read 2 Done Sent Done false 1021 Acked Write 6 Done Sent Done false 1022 Acked Read 8 Done Sent Done false 1023 Acked Read 8 Done Sent Done false 1024 Acked Read 4 Done Sent Done false 1025 Acked Read 10 Done Sent Done false 1026 Acked Write 1 Done Sent Done false 1027 Acked Write 2 Done Sent Done false 1028 Acked Write 2 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 25 2 29 Done 28 5 28 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) None Some(JobId(1017)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:57:29.342736792Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1592} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:29.342918355Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1592,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 194 194 194 F F F 1 020-039 110 110 110 189 189 189 F F F 2 040-059 107 107 107 183 183 183 F F F 3 060-079 114 98 114 198 171 198 F F F <--- 4 080-099 102 102 102 176 176 176 F F F 5 100-119 113 113 113 196 196 196 F F F 6 120-139 113 113 113 196 196 196 F F F 7 140-159 111 111 111 193 193 193 F F F 8 160-179 114 111 114 198 193 198 F F F <--- 9 180-199 112 112 112 194 194 194 F F F 10 200-219 112 112 112 194 194 194 F F F 11 220-239 110 110 110 189 189 189 F F F 12 240-259 112 112 112 194 194 194 F F F 13 260-279 112 112 112 194 194 194 F F F 14 280-299 105 105 105 181 181 181 T F T <--- 15 300-319 109 109 109 186 186 186 T F T <--- 16 320-339 112 112 112 194 194 194 F F F 17 340-359 112 112 112 194 194 194 T F T <--- 18 360-379 112 112 112 194 194 194 F F F 19 380-399 110 110 110 189 189 189 F F F 20 400-419 114 110 114 197 187 197 F T F <--- 21 420-439 114 110 114 197 187 197 T T T <--- 22 440-459 114 108 114 198 185 198 F F F <--- 23 460-479 114 110 114 198 187 198 F F F <--- 24 480-499 114 111 114 198 193 198 F F F <--- 25 500-519 114 113 114 198 196 198 T F T <--- 26 520-539 114 113 114 198 196 198 F F F <--- 27 540-559 112 112 112 194 194 194 F F F 28 560-579 112 112 112 194 194 194 F F F 29 580-599 112 112 112 194 194 194 T F T <--- Max gen: 114, Max flush: 198 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 57 Verifying data now No change after verify Loop: 57 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 194 194 194 F F F 1 020-039 110 110 110 189 189 189 F F F 2 040-059 107 107 107 183 183 183 F F F 3 060-079 114 114 114 198 198 198 F F F 4 080-099 102 102 102 176 176 176 F F F 5 100-119 113 113 113 196 196 196 F F F 6 120-139 113 113 113 196 196 196 F F F 7 140-159 111 111 111 193 193 193 F F F 8 160-179 114 114 114 198 198 198 F F F 9 180-199 112 112 112 194 194 194 F F F 10 200-219 112 112 112 194 194 194 F F F 11 220-239 110 110 110 189 189 189 F F F 12 240-259 112 112 112 194 194 194 F F F 13 260-279 112 112 112 194 194 194 F F F 14 280-299 115 115 115 199 199 199 F F F 15 300-319 115 115 115 199 199 199 F F F 16 320-339 112 112 112 194 194 194 F F F 17 340-359 115 115 115 199 199 199 F F F 18 360-379 112 112 112 194 194 194 F F F 19 380-399 110 110 110 189 189 189 F F F 20 400-419 114 114 114 197 197 197 F F F 21 420-439 115 115 115 199 199 199 F F F 22 440-459 114 114 114 198 198 198 F F F 23 460-479 114 114 114 198 198 198 F F F 24 480-499 114 114 114 198 198 198 F F F 25 500-519 115 115 115 199 199 199 F F F 26 520-539 114 114 114 198 198 198 F F F 27 540-559 112 112 112 194 194 194 F F F 28 560-579 112 112 112 194 194 194 F F F 29 580-599 115 115 115 199 199 199 F F F Max gen: 115, Max flush: 199 Begin loop 58 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:33.590896547Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1600,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:33.599033847Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1600,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:33.599318085Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1600} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:33.599364408Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1600} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:33.599393504Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1600} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:33.599406512Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1600} {"msg":"Crucible a18561e5-4fdf-4f1f-9a23-432abf1f9cd1 has session id: e65c7fd7-fdaa-4585-af75-2cd078940ec7","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:33.599715346Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1600,"session_id":"e65c7fd7-fdaa-4585-af75-2cd078940ec7"} {"msg":"Upstairs opts: Upstairs UUID: a18561e5-4fdf-4f1f-9a23-432abf1f9cd1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:33.599736273Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1600,"session_id":"e65c7fd7-fdaa-4585-af75-2cd078940ec7"} {"msg":"Crucible stats registered with UUID: a18561e5-4fdf-4f1f-9a23-432abf1f9cd1","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:33.599746462Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1600,"session_id":"e65c7fd7-fdaa-4585-af75-2cd078940ec7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:57:33.59975545Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1600,"session_id":"e65c7fd7-fdaa-4585-af75-2cd078940ec7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:33.599910758Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1600,"":"io task","client":"2","":"downstairs","session_id":"e65c7fd7-fdaa-4585-af75-2cd078940ec7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:33.599942863Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1600,"":"io task","client":"0","":"downstairs","session_id":"e65c7fd7-fdaa-4585-af75-2cd078940ec7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:33.599953921Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1600,"":"io task","client":"1","":"downstairs","session_id":"e65c7fd7-fdaa-4585-af75-2cd078940ec7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:33.600082862Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1600,"task":"crutest"} {"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:33.600092621Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1600} {"msg":"a18561e5-4fdf-4f1f-9a23-432abf1f9cd1 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:33.600127795Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1600,"session_id":"e65c7fd7-fdaa-4585-af75-2cd078940ec7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:33.600137474Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1600,"client":"0","":"downstairs","session_id":"e65c7fd7-fdaa-4585-af75-2cd078940ec7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:33.600147203Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1600,"client":"1","":"downstairs","session_id":"e65c7fd7-fdaa-4585-af75-2cd078940ec7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:33.600159401Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1600,"client":"2","":"downstairs","session_id":"e65c7fd7-fdaa-4585-af75-2cd078940ec7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:33.600169129Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1600,"":"io task","client":"2","":"downstairs","session_id":"e65c7fd7-fdaa-4585-af75-2cd078940ec7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:33.600179308Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1600,"":"io task","client":"0","":"downstairs","session_id":"e65c7fd7-fdaa-4585-af75-2cd078940ec7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:33.600189366Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1600,"":"io task","client":"1","":"downstairs","session_id":"e65c7fd7-fdaa-4585-af75-2cd078940ec7"} {"msg":"ds_connection connected from Ok(127.0.0.1:64191)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:33.600226621Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1600,"":"io task","client":"2","":"downstairs","session_id":"e65c7fd7-fdaa-4585-af75-2cd078940ec7"} {"msg":"ds_connection connected from Ok(127.0.0.1:53756)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:33.600237119Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1600,"":"io task","client":"1","":"downstairs","session_id":"e65c7fd7-fdaa-4585-af75-2cd078940ec7"} {"msg":"ds_connection connected from Ok(127.0.0.1:38775)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:33.600278463Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1600,"":"io task","client":"0","":"downstairs","session_id":"e65c7fd7-fdaa-4585-af75-2cd078940ec7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:33.600941585Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1600,"client":"1","":"downstairs","session_id":"e65c7fd7-fdaa-4585-af75-2cd078940ec7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:33.600967492Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1600,"client":"0","":"downstairs","session_id":"e65c7fd7-fdaa-4585-af75-2cd078940ec7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:33.601030622Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1600,"client":"2","":"downstairs","session_id":"e65c7fd7-fdaa-4585-af75-2cd078940ec7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:33.60124677Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1600,"session_id":"e65c7fd7-fdaa-4585-af75-2cd078940ec7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:33.601266537Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1600,"session_id":"e65c7fd7-fdaa-4585-af75-2cd078940ec7"} {"msg":"[0]R flush_numbers[0..12]: [194, 189, 183, 198, 176, 196, 196, 193, 198, 194, 194, 189]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:33.601341416Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1600,"":"downstairs","":"downstairs","session_id":"e65c7fd7-fdaa-4585-af75-2cd078940ec7"} {"msg":"[0]R generation[0..12]: [112, 110, 107, 114, 102, 113, 113, 111, 114, 112, 112, 110]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:33.601352755Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1600,"":"downstairs","":"downstairs","session_id":"e65c7fd7-fdaa-4585-af75-2cd078940ec7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:33.601366063Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1600,"":"downstairs","":"downstairs","session_id":"e65c7fd7-fdaa-4585-af75-2cd078940ec7"} {"msg":"[1]R flush_numbers[0..12]: [194, 189, 183, 198, 176, 196, 196, 193, 198, 194, 194, 189]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:33.601376881Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1600,"":"downstairs","":"downstairs","session_id":"e65c7fd7-fdaa-4585-af75-2cd078940ec7"} {"msg":"[1]R generation[0..12]: [112, 110, 107, 114, 102, 113, 113, 111, 114, 112, 112, 110]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:33.6013874Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1600,"":"downstairs","":"downstairs","session_id":"e65c7fd7-fdaa-4585-af75-2cd078940ec7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:33.601398658Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1600,"":"downstairs","":"downstairs","session_id":"e65c7fd7-fdaa-4585-af75-2cd078940ec7"} {"msg":"[2]R flush_numbers[0..12]: [194, 189, 183, 198, 176, 196, 196, 193, 198, 194, 194, 189]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:33.601409406Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1600,"":"downstairs","":"downstairs","session_id":"e65c7fd7-fdaa-4585-af75-2cd078940ec7"} {"msg":"[2]R generation[0..12]: [112, 110, 107, 114, 102, 113, 113, 111, 114, 112, 112, 110]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:33.601420095Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1600,"":"downstairs","":"downstairs","session_id":"e65c7fd7-fdaa-4585-af75-2cd078940ec7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:33.601430993Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1600,"":"downstairs","":"downstairs","session_id":"e65c7fd7-fdaa-4585-af75-2cd078940ec7"} {"msg":"Max found gen is 116","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:33.601440342Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1600,"":"downstairs","":"downstairs","session_id":"e65c7fd7-fdaa-4585-af75-2cd078940ec7"} {"msg":"Generation requested: 116 >= found:116","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:33.60145008Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1600,"":"downstairs","":"downstairs","session_id":"e65c7fd7-fdaa-4585-af75-2cd078940ec7"} {"msg":"Next flush: 200","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:33.601459619Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1600,"":"downstairs","":"downstairs","session_id":"e65c7fd7-fdaa-4585-af75-2cd078940ec7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:33.601469078Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1600,"":"downstairs","":"downstairs","session_id":"e65c7fd7-fdaa-4585-af75-2cd078940ec7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:33.601478926Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1600,"session_id":"e65c7fd7-fdaa-4585-af75-2cd078940ec7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:33.601487575Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1600,"client":"0","":"downstairs","session_id":"e65c7fd7-fdaa-4585-af75-2cd078940ec7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:33.601497133Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1600,"client":"1","":"downstairs","session_id":"e65c7fd7-fdaa-4585-af75-2cd078940ec7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:33.601506752Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1600,"client":"2","":"downstairs","session_id":"e65c7fd7-fdaa-4585-af75-2cd078940ec7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:33.60151801Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1600,"session_id":"e65c7fd7-fdaa-4585-af75-2cd078940ec7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:33.601526959Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1600,"session_id":"e65c7fd7-fdaa-4585-af75-2cd078940ec7"} {"msg":"a18561e5-4fdf-4f1f-9a23-432abf1f9cd1 is now active with session: e65c7fd7-fdaa-4585-af75-2cd078940ec7","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:33.601536708Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1600,"session_id":"e65c7fd7-fdaa-4585-af75-2cd078940ec7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:33.601545456Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1600,"session_id":"e65c7fd7-fdaa-4585-af75-2cd078940ec7"} {"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:33.601554545Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1600} {"msg":"Activated sub_volume a18561e5-4fdf-4f1f-9a23-432abf1f9cd1","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:33.601562534Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1600} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:33.601570623Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1600,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:33.601892775Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1600,"task":"crutest"} 01/30 Write block 272 len 20480 data: 11 10 11 12 10 02/30 Flush 03/30 Write block 74 len 20480 data: 7 7 7 7 7 04/30 Read block 214 len 12288 05/30 Flush 06/30 Write block 503 len 12288 data: 9 7 7 07/30 Flush 08/30 Flush 09/30 Write block 289 len 20480 data: 8 9 7 7 6 10/30 Flush 11/30 Write block 520 len 20480 data: 11 11 8 6 5 12/30 Write block 54 len 32768 data: 9 10 12 8 7 5 7 6 13/30 Read block 78 len 32768 14/30 Write block 96 len 32768 data: 11 10 10 10 15 16 10 12 15/30 Read block 52 len 40960 16/30 Read block 158 len 28672 17/30 Read block 251 len 40960 18/30 Flush 19/30 Write block 366 len 40960 data: 14 12 12 10 11 9 11 9 8 9 20/30 Read block 507 len 40960 21/30 Read block 401 len 12288 22/30 Read block 452 len 12288 23/30 Read block 555 len 32768 24/30 Read block 242 len 24576 25/30 Read block 299 len 40960 26/30 Write block 161 len 40960 data: 7 7 6 6 8 8 8 6 6 5 27/30 Write block 423 len 36864 data: 11 13 13 9 13 12 11 10 8 28/30 Read block 492 len 24576 29/30 Write block 408 len 8192 data: 10 10 30/30 Read block 360 len 40960 ---------------------------------------------------------------- Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Sent false 1001 Acked Flush 0 Done Done Sent false 1002 Acked Write 5 Done Done Sent false 1003 Acked Read 3 Done Done Sent false 1004 Acked Flush 0 Done Done Sent false 1005 Acked Write 3 Done Done Sent false 1006 Acked Flush 0 Done Done Sent false 1007 Acked Flush 0 Done Done Sent false 1008 Acked Write 5 Done Done Sent false 1009 Acked Flush 0 Done Done Sent false 1010 Acked Write 5 Done Done Sent false 1011 Acked Write 8 Done Done Sent false 1012 Acked Read 8 Done Done Sent false 1013 Acked Write 8 Done Done Sent false 1014 Acked Read 10 Done Done Sent false 1015 Acked Read 7 Done Done Sent false 1016 Acked Read 10 Done Done Sent false 1017 Acked Flush 0 Done Done Sent false 1018 Acked Write 10 Done Done Sent false 1019 Acked Read 10 Done Done Sent false 1020 Acked Read 3 Done Done Sent false 1021 Acked Read 3 Done Done Sent false 1022 Acked Read 8 Done Done Sent false 1023 Acked Read 6 Done Done Sent false 1024 Acked Read 10 Done Done Sent false 1025 Acked Write 10 Done Done Sent false 1026 Acked Write 9 Done Done Sent false 1027 Acked Read 6 Done Done Sent false 1028 Acked Write 2 Done Done Sent false 1029 Acked Read 10 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 30 30 Done 30 30 0 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:57:33.611404532Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1600} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:33.611588645Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1600,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 194 194 194 F F F 1 020-039 110 110 110 189 189 189 F F F 2 040-059 116 116 107 205 205 183 F F F <--- 3 060-079 116 116 114 205 205 198 F F F <--- 4 080-099 116 116 102 205 205 176 F F F <--- 5 100-119 116 116 113 205 205 196 F F F <--- 6 120-139 113 113 113 196 196 196 F F F 7 140-159 111 111 111 193 193 193 F F F 8 160-179 114 114 114 198 198 198 T T F <--- 9 180-199 112 112 112 194 194 194 F F F 10 200-219 112 112 112 194 194 194 F F F 11 220-239 110 110 110 189 189 189 F F F 12 240-259 112 112 112 194 194 194 F F F 13 260-279 116 116 112 200 200 194 F F F <--- 14 280-299 116 116 115 204 204 199 F F F <--- 15 300-319 115 115 115 199 199 199 F F F 16 320-339 112 112 112 194 194 194 F F F 17 340-359 115 115 115 199 199 199 F F F 18 360-379 112 112 112 194 194 194 T T F <--- 19 380-399 110 110 110 189 189 189 F F F 20 400-419 114 114 114 197 197 197 T T F <--- 21 420-439 115 115 115 199 199 199 T T F <--- 22 440-459 114 114 114 198 198 198 F F F 23 460-479 114 114 114 198 198 198 F F F 24 480-499 114 114 114 198 198 198 F F F 25 500-519 116 116 115 202 202 199 F F F <--- 26 520-539 116 116 114 205 205 198 F F F <--- 27 540-559 112 112 112 194 194 194 F F F 28 560-579 112 112 112 194 194 194 F F F 29 580-599 115 115 115 199 199 199 F F F Max gen: 116, Max flush: 205 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 58 Verifying data now No change after verify Loop: 58 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 194 194 194 F F F 1 020-039 110 110 110 189 189 189 F F F 2 040-059 116 116 116 205 205 205 F F F 3 060-079 116 116 116 205 205 205 F F F 4 080-099 116 116 116 205 205 205 F F F 5 100-119 116 116 116 205 205 205 F F F 6 120-139 113 113 113 196 196 196 F F F 7 140-159 111 111 111 193 193 193 F F F 8 160-179 117 117 117 206 206 206 F F F 9 180-199 112 112 112 194 194 194 F F F 10 200-219 112 112 112 194 194 194 F F F 11 220-239 110 110 110 189 189 189 F F F 12 240-259 112 112 112 194 194 194 F F F 13 260-279 116 116 116 200 200 200 F F F 14 280-299 116 116 116 204 204 204 F F F 15 300-319 115 115 115 199 199 199 F F F 16 320-339 112 112 112 194 194 194 F F F 17 340-359 115 115 115 199 199 199 F F F 18 360-379 117 117 117 206 206 206 F F F 19 380-399 110 110 110 189 189 189 F F F 20 400-419 117 117 117 206 206 206 F F F 21 420-439 117 117 117 206 206 206 F F F 22 440-459 114 114 114 198 198 198 F F F 23 460-479 114 114 114 198 198 198 F F F 24 480-499 114 114 114 198 198 198 F F F 25 500-519 116 116 116 202 202 202 F F F 26 520-539 116 116 116 205 205 205 F F F 27 540-559 112 112 112 194 194 194 F F F 28 560-579 112 112 112 194 194 194 F F F 29 580-599 115 115 115 199 199 199 F F F Max gen: 117, Max flush: 206 Begin loop 59 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:37.856845957Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1608,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:57:37.863959738Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1608,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:37.864008181Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1608,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:37.87085702Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1608,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:37.871107463Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1608} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:37.871136869Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1608} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:37.871145488Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1608} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:37.871156886Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1608} {"msg":"Crucible fde75389-6f1f-4a3c-86a0-026f60c0c382 has session id: 316492e8-4ac0-4202-89ce-e06d31169daa","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:37.871486817Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1608,"session_id":"316492e8-4ac0-4202-89ce-e06d31169daa"} {"msg":"Upstairs opts: Upstairs UUID: fde75389-6f1f-4a3c-86a0-026f60c0c382, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:37.871516873Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1608,"session_id":"316492e8-4ac0-4202-89ce-e06d31169daa"} {"msg":"Crucible stats registered with UUID: fde75389-6f1f-4a3c-86a0-026f60c0c382","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:37.871527981Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1608,"session_id":"316492e8-4ac0-4202-89ce-e06d31169daa"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:57:37.87153702Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1608,"session_id":"316492e8-4ac0-4202-89ce-e06d31169daa"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:37.871641385Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1608,"":"io task","client":"0","":"downstairs","session_id":"316492e8-4ac0-4202-89ce-e06d31169daa"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:37.871658062Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1608,"":"io task","client":"2","":"downstairs","session_id":"316492e8-4ac0-4202-89ce-e06d31169daa"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:37.87167356Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1608,"":"io task","client":"1","":"downstairs","session_id":"316492e8-4ac0-4202-89ce-e06d31169daa"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:37.871854723Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1608,"task":"crutest"} {"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:37.871869021Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1608} {"msg":"fde75389-6f1f-4a3c-86a0-026f60c0c382 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:37.871880599Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1608,"session_id":"316492e8-4ac0-4202-89ce-e06d31169daa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:37.871889438Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1608,"client":"0","":"downstairs","session_id":"316492e8-4ac0-4202-89ce-e06d31169daa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:37.871899037Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1608,"client":"1","":"downstairs","session_id":"316492e8-4ac0-4202-89ce-e06d31169daa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:37.871908885Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1608,"client":"2","":"downstairs","session_id":"316492e8-4ac0-4202-89ce-e06d31169daa"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:37.871918444Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1608,"":"io task","client":"2","":"downstairs","session_id":"316492e8-4ac0-4202-89ce-e06d31169daa"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:37.871928582Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1608,"":"io task","client":"0","":"downstairs","session_id":"316492e8-4ac0-4202-89ce-e06d31169daa"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:37.871938621Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1608,"":"io task","client":"1","":"downstairs","session_id":"316492e8-4ac0-4202-89ce-e06d31169daa"} {"msg":"ds_connection connected from Ok(127.0.0.1:32847)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:37.871974505Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1608,"":"io task","client":"1","":"downstairs","session_id":"316492e8-4ac0-4202-89ce-e06d31169daa"} {"msg":"ds_connection connected from Ok(127.0.0.1:48359)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:37.87201299Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1608,"":"io task","client":"0","":"downstairs","session_id":"316492e8-4ac0-4202-89ce-e06d31169daa"} {"msg":"ds_connection connected from Ok(127.0.0.1:44495)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:37.87207971Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1608,"":"io task","client":"2","":"downstairs","session_id":"316492e8-4ac0-4202-89ce-e06d31169daa"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:37.872641477Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1608,"client":"1","":"downstairs","session_id":"316492e8-4ac0-4202-89ce-e06d31169daa"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:37.872659614Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1608,"client":"0","":"downstairs","session_id":"316492e8-4ac0-4202-89ce-e06d31169daa"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:37.872670723Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1608,"client":"2","":"downstairs","session_id":"316492e8-4ac0-4202-89ce-e06d31169daa"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:37.872983826Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1608,"session_id":"316492e8-4ac0-4202-89ce-e06d31169daa"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:37.87302562Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1608,"session_id":"316492e8-4ac0-4202-89ce-e06d31169daa"} {"msg":"[0]R flush_numbers[0..12]: [194, 189, 205, 205, 205, 205, 196, 193, 206, 194, 194, 189]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:37.873105908Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1608,"":"downstairs","":"downstairs","session_id":"316492e8-4ac0-4202-89ce-e06d31169daa"} {"msg":"[0]R generation[0..12]: [112, 110, 116, 116, 116, 116, 113, 111, 117, 112, 112, 110]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:37.873117137Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1608,"":"downstairs","":"downstairs","session_id":"316492e8-4ac0-4202-89ce-e06d31169daa"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:37.873128125Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1608,"":"downstairs","":"downstairs","session_id":"316492e8-4ac0-4202-89ce-e06d31169daa"} {"msg":"[1]R flush_numbers[0..12]: [194, 189, 205, 205, 205, 205, 196, 193, 206, 194, 194, 189]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:37.873138724Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1608,"":"downstairs","":"downstairs","session_id":"316492e8-4ac0-4202-89ce-e06d31169daa"} {"msg":"[1]R generation[0..12]: [112, 110, 116, 116, 116, 116, 113, 111, 117, 112, 112, 110]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:37.873149152Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1608,"":"downstairs","":"downstairs","session_id":"316492e8-4ac0-4202-89ce-e06d31169daa"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:37.87315999Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1608,"":"downstairs","":"downstairs","session_id":"316492e8-4ac0-4202-89ce-e06d31169daa"} {"msg":"[2]R flush_numbers[0..12]: [194, 189, 205, 205, 205, 205, 196, 193, 206, 194, 194, 189]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:37.873173139Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1608,"":"downstairs","":"downstairs","session_id":"316492e8-4ac0-4202-89ce-e06d31169daa"} {"msg":"[2]R generation[0..12]: [112, 110, 116, 116, 116, 116, 113, 111, 117, 112, 112, 110]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:37.873183867Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1608,"":"downstairs","":"downstairs","session_id":"316492e8-4ac0-4202-89ce-e06d31169daa"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:37.873194915Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1608,"":"downstairs","":"downstairs","session_id":"316492e8-4ac0-4202-89ce-e06d31169daa"} {"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:37.873204474Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1608,"":"downstairs","":"downstairs","session_id":"316492e8-4ac0-4202-89ce-e06d31169daa"} {"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:37.873214322Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1608,"":"downstairs","":"downstairs","session_id":"316492e8-4ac0-4202-89ce-e06d31169daa"} {"msg":"Next flush: 207","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:37.873223771Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1608,"":"downstairs","":"downstairs","session_id":"316492e8-4ac0-4202-89ce-e06d31169daa"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:37.87323333Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1608,"":"downstairs","":"downstairs","session_id":"316492e8-4ac0-4202-89ce-e06d31169daa"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:37.873243188Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1608,"session_id":"316492e8-4ac0-4202-89ce-e06d31169daa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:37.873251897Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1608,"client":"0","":"downstairs","session_id":"316492e8-4ac0-4202-89ce-e06d31169daa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:37.873261885Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1608,"client":"1","":"downstairs","session_id":"316492e8-4ac0-4202-89ce-e06d31169daa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:37.873271744Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1608,"client":"2","":"downstairs","session_id":"316492e8-4ac0-4202-89ce-e06d31169daa"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:37.873281763Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1608,"session_id":"316492e8-4ac0-4202-89ce-e06d31169daa"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:37.873290911Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1608,"session_id":"316492e8-4ac0-4202-89ce-e06d31169daa"} {"msg":"fde75389-6f1f-4a3c-86a0-026f60c0c382 is now active with session: 316492e8-4ac0-4202-89ce-e06d31169daa","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:37.87330099Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1608,"session_id":"316492e8-4ac0-4202-89ce-e06d31169daa"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:37.873309668Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1608,"session_id":"316492e8-4ac0-4202-89ce-e06d31169daa"} {"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:37.873320197Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1608} {"msg":"Activated sub_volume fde75389-6f1f-4a3c-86a0-026f60c0c382","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:37.873328136Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1608} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:37.873336194Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1608,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:37.873648268Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1608,"task":"crutest"} 01/30 Write block 117 len 36864 data: 10 9 10 10 12 10 11 10 9 02/30 Write block 46 len 28672 data: 14 13 12 10 10 8 6 03/30 Write block 104 len 20480 data: 9 10 7 8 8 04/30 Write block 459 len 12288 data: 8 7 7 05/30 Read block 457 len 16384 06/30 Read block 270 len 24576 07/30 Write block 412 len 36864 data: 15 16 12 12 13 8 6 7 7 08/30 Read block 227 len 32768 09/30 Write block 410 len 8192 data: 13 18 10/30 Read block 373 len 32768 11/30 Read block 326 len 24576 12/30 Read block 271 len 32768 13/30 Write block 269 len 16384 data: 8 11 13 12 14/30 Flush 15/30 Write block 474 len 28672 data: 9 11 10 11 13 13 12 16/30 Write block 301 len 4096 data: 9 17/30 Read block 560 len 20480 18/30 Read block 500 len 8192 19/30 Flush 20/30 Write block 169 len 4096 data: 7 21/30 Read block 396 len 40960 22/30 Read block 133 len 20480 23/30 Read block 247 len 24576 24/30 Write block 158 len 16384 data: 7 7 7 8 25/30 Write block 394 len 32768 data: 7 8 6 7 7 6 6 4 26/30 Read block 456 len 40960 27/30 Write block 118 len 28672 data: 10 11 11 13 11 12 11 28/30 Read block 202 len 32768 29/30 Write block 88 len 36864 data: 12 11 11 10 10 11 9 11 12 30/30 Read block 427 len 36864 ---------------------------------------------------------------- Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Sent Done Done false 1001 Acked Write 7 Sent Done Done false 1002 Acked Write 5 Sent Done Done false 1003 Acked Write 3 Sent Done Done false 1004 Acked Read 4 Sent Done Done false 1005 Acked Read 6 Sent Done Done false 1006 Acked Write 9 Sent Done Done false 1007 Acked Read 8 Sent Done Done false 1008 Acked Write 2 Sent Done Done false 1009 Acked Read 8 Sent Done Done false 1010 Acked Read 6 Sent Done Done false 1011 Acked Read 8 Sent Done Done false 1012 Acked Write 4 Sent Done Done false 1013 Acked Flush 0 Sent Done Done false 1014 Acked Write 7 Sent Done Done false 1015 Acked Write 1 Sent Done Done false 1016 Acked Read 5 Sent Done Done false 1017 Acked Read 2 Sent Done Done false 1018 Acked Flush 0 Sent Done Done false 1019 Acked Write 1 Sent Done Done false 1020 Acked Read 10 Sent Done Done false 1021 Acked Read 5 Sent Done Done false 1022 Acked Read 6 Sent Done Done false 1023 Acked Write 4 Sent Done Done false 1024 Acked Write 8 Sent Done Done false 1025 Acked Read 10 Sent Done Done false 1026 Acked Write 7 Sent Done Done false 1027 Acked Read 8 Sent Done Done false 1028 Acked Write 9 Sent Done Done false 1029 Acked Read 9 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 0 1 31 Done 0 30 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:57:37.882319079Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1608} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:37.882499952Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1608,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 194 194 194 F F F 1 020-039 110 110 110 189 189 189 F F F 2 040-059 116 118 118 205 207 207 F F F <--- 3 060-079 116 116 116 205 205 205 F F F 4 080-099 116 116 116 205 205 205 F T T <--- 5 100-119 116 118 118 205 207 207 F T T <--- 6 120-139 113 118 118 196 207 207 F T T <--- 7 140-159 111 111 111 193 193 193 F T T <--- 8 160-179 117 117 117 206 206 206 F T T <--- 9 180-199 112 112 112 194 194 194 F F F 10 200-219 112 112 112 194 194 194 F F F 11 220-239 110 110 110 189 189 189 F F F 12 240-259 112 112 112 194 194 194 F F F 13 260-279 116 118 118 200 207 207 F F F <--- 14 280-299 116 116 116 204 204 204 F F F 15 300-319 115 118 118 199 208 208 F F F <--- 16 320-339 112 112 112 194 194 194 F F F 17 340-359 115 115 115 199 199 199 F F F 18 360-379 117 117 117 206 206 206 F F F 19 380-399 110 110 110 189 189 189 F T T <--- 20 400-419 117 118 118 206 207 207 F T T <--- 21 420-439 117 118 118 206 207 207 F F F <--- 22 440-459 114 118 118 198 207 207 F F F <--- 23 460-479 114 118 118 198 208 208 F F F <--- 24 480-499 114 118 118 198 208 208 F F F <--- 25 500-519 116 116 116 202 202 202 F F F 26 520-539 116 116 116 205 205 205 F F F 27 540-559 112 112 112 194 194 194 F F F 28 560-579 112 112 112 194 194 194 F F F 29 580-599 115 115 115 199 199 199 F F F Max gen: 118, Max flush: 208 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 59 Verifying data now No change after verify Loop: 59 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 194 194 194 F F F 1 020-039 110 110 110 189 189 189 F F F 2 040-059 118 118 118 207 207 207 F F F 3 060-079 116 116 116 205 205 205 F F F 4 080-099 119 119 119 209 209 209 F F F 5 100-119 119 119 119 209 209 209 F F F 6 120-139 119 119 119 209 209 209 F F F 7 140-159 119 119 119 209 209 209 F F F 8 160-179 119 119 119 209 209 209 F F F 9 180-199 112 112 112 194 194 194 F F F 10 200-219 112 112 112 194 194 194 F F F 11 220-239 110 110 110 189 189 189 F F F 12 240-259 112 112 112 194 194 194 F F F 13 260-279 118 118 118 207 207 207 F F F 14 280-299 116 116 116 204 204 204 F F F 15 300-319 118 118 118 208 208 208 F F F 16 320-339 112 112 112 194 194 194 F F F 17 340-359 115 115 115 199 199 199 F F F 18 360-379 117 117 117 206 206 206 F F F 19 380-399 119 119 119 209 209 209 F F F 20 400-419 119 119 119 209 209 209 F F F 21 420-439 118 118 118 207 207 207 F F F 22 440-459 118 118 118 207 207 207 F F F 23 460-479 118 118 118 208 208 208 F F F 24 480-499 118 118 118 208 208 208 F F F 25 500-519 116 116 116 202 202 202 F F F 26 520-539 116 116 116 205 205 205 F F F 27 540-559 112 112 112 194 194 194 F F F 28 560-579 112 112 112 194 194 194 F F F 29 580-599 115 115 115 199 199 199 F F F Max gen: 119, Max flush: 209 Begin loop 60 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:42.138432111Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1616,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:42.146703001Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1616,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:42.147010635Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1616} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:42.147037541Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1616} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:42.14704636Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1616} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:42.147058188Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1616} {"msg":"Crucible 24749021-7c2f-4c30-b784-547e1c3475c8 has session id: f532b587-630d-41e1-9bf2-56ed00c5bd35","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:42.147372632Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1616,"session_id":"f532b587-630d-41e1-9bf2-56ed00c5bd35"} {"msg":"Upstairs opts: Upstairs UUID: 24749021-7c2f-4c30-b784-547e1c3475c8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:42.147402157Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1616,"session_id":"f532b587-630d-41e1-9bf2-56ed00c5bd35"} {"msg":"Crucible stats registered with UUID: 24749021-7c2f-4c30-b784-547e1c3475c8","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:42.147413186Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1616,"session_id":"f532b587-630d-41e1-9bf2-56ed00c5bd35"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:57:42.147422514Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1616,"session_id":"f532b587-630d-41e1-9bf2-56ed00c5bd35"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:42.147566123Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1616,"":"io task","client":"1","":"downstairs","session_id":"f532b587-630d-41e1-9bf2-56ed00c5bd35"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:42.147582781Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1616,"":"io task","client":"0","":"downstairs","session_id":"f532b587-630d-41e1-9bf2-56ed00c5bd35"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:42.147593129Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1616,"":"io task","client":"2","":"downstairs","session_id":"f532b587-630d-41e1-9bf2-56ed00c5bd35"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:42.147747036Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1616,"task":"crutest"} {"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:42.147761944Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1616} {"msg":"24749021-7c2f-4c30-b784-547e1c3475c8 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:42.147799849Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1616,"session_id":"f532b587-630d-41e1-9bf2-56ed00c5bd35"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:42.147809217Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1616,"client":"0","":"downstairs","session_id":"f532b587-630d-41e1-9bf2-56ed00c5bd35"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:42.147819086Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1616,"client":"1","":"downstairs","session_id":"f532b587-630d-41e1-9bf2-56ed00c5bd35"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:42.147828664Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1616,"client":"2","":"downstairs","session_id":"f532b587-630d-41e1-9bf2-56ed00c5bd35"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:42.147838543Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1616,"":"io task","client":"2","":"downstairs","session_id":"f532b587-630d-41e1-9bf2-56ed00c5bd35"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:42.147848462Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1616,"":"io task","client":"0","":"downstairs","session_id":"f532b587-630d-41e1-9bf2-56ed00c5bd35"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:42.14785824Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1616,"":"io task","client":"1","":"downstairs","session_id":"f532b587-630d-41e1-9bf2-56ed00c5bd35"} {"msg":"ds_connection connected from Ok(127.0.0.1:62735)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:42.147923211Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1616,"":"io task","client":"2","":"downstairs","session_id":"f532b587-630d-41e1-9bf2-56ed00c5bd35"} {"msg":"ds_connection connected from Ok(127.0.0.1:63487)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:42.147934319Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1616,"":"io task","client":"0","":"downstairs","session_id":"f532b587-630d-41e1-9bf2-56ed00c5bd35"} {"msg":"ds_connection connected from Ok(127.0.0.1:41457)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:42.148021306Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1616,"":"io task","client":"1","":"downstairs","session_id":"f532b587-630d-41e1-9bf2-56ed00c5bd35"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:42.148563146Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1616,"client":"1","":"downstairs","session_id":"f532b587-630d-41e1-9bf2-56ed00c5bd35"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:42.148587532Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1616,"client":"0","":"downstairs","session_id":"f532b587-630d-41e1-9bf2-56ed00c5bd35"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:42.148706485Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1616,"client":"2","":"downstairs","session_id":"f532b587-630d-41e1-9bf2-56ed00c5bd35"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:42.148898437Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1616,"session_id":"f532b587-630d-41e1-9bf2-56ed00c5bd35"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:42.148918514Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1616,"session_id":"f532b587-630d-41e1-9bf2-56ed00c5bd35"} {"msg":"[0]R flush_numbers[0..12]: [194, 189, 207, 205, 209, 209, 209, 209, 209, 194, 194, 189]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:42.149018229Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1616,"":"downstairs","":"downstairs","session_id":"f532b587-630d-41e1-9bf2-56ed00c5bd35"} {"msg":"[0]R generation[0..12]: [112, 110, 118, 116, 119, 119, 119, 119, 119, 112, 112, 110]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:42.149029957Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1616,"":"downstairs","":"downstairs","session_id":"f532b587-630d-41e1-9bf2-56ed00c5bd35"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:42.149041226Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1616,"":"downstairs","":"downstairs","session_id":"f532b587-630d-41e1-9bf2-56ed00c5bd35"} {"msg":"[1]R flush_numbers[0..12]: [194, 189, 207, 205, 209, 209, 209, 209, 209, 194, 194, 189]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:42.149051924Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1616,"":"downstairs","":"downstairs","session_id":"f532b587-630d-41e1-9bf2-56ed00c5bd35"} {"msg":"[1]R generation[0..12]: [112, 110, 118, 116, 119, 119, 119, 119, 119, 112, 112, 110]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:42.149062512Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1616,"":"downstairs","":"downstairs","session_id":"f532b587-630d-41e1-9bf2-56ed00c5bd35"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:42.149073571Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1616,"":"downstairs","":"downstairs","session_id":"f532b587-630d-41e1-9bf2-56ed00c5bd35"} {"msg":"[2]R flush_numbers[0..12]: [194, 189, 207, 205, 209, 209, 209, 209, 209, 194, 194, 189]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:42.149084149Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1616,"":"downstairs","":"downstairs","session_id":"f532b587-630d-41e1-9bf2-56ed00c5bd35"} {"msg":"[2]R generation[0..12]: [112, 110, 118, 116, 119, 119, 119, 119, 119, 112, 112, 110]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:42.149094728Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1616,"":"downstairs","":"downstairs","session_id":"f532b587-630d-41e1-9bf2-56ed00c5bd35"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:42.149105746Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1616,"":"downstairs","":"downstairs","session_id":"f532b587-630d-41e1-9bf2-56ed00c5bd35"} {"msg":"Max found gen is 120","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:42.149115295Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1616,"":"downstairs","":"downstairs","session_id":"f532b587-630d-41e1-9bf2-56ed00c5bd35"} {"msg":"Generation requested: 120 >= found:120","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:42.149127263Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1616,"":"downstairs","":"downstairs","session_id":"f532b587-630d-41e1-9bf2-56ed00c5bd35"} {"msg":"Next flush: 210","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:42.149136731Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1616,"":"downstairs","":"downstairs","session_id":"f532b587-630d-41e1-9bf2-56ed00c5bd35"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:42.14914617Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1616,"":"downstairs","":"downstairs","session_id":"f532b587-630d-41e1-9bf2-56ed00c5bd35"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:42.149156049Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1616,"session_id":"f532b587-630d-41e1-9bf2-56ed00c5bd35"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:42.149164867Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1616,"client":"0","":"downstairs","session_id":"f532b587-630d-41e1-9bf2-56ed00c5bd35"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:42.149174616Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1616,"client":"1","":"downstairs","session_id":"f532b587-630d-41e1-9bf2-56ed00c5bd35"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:42.149184134Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1616,"client":"2","":"downstairs","session_id":"f532b587-630d-41e1-9bf2-56ed00c5bd35"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:42.149193873Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1616,"session_id":"f532b587-630d-41e1-9bf2-56ed00c5bd35"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:42.149203082Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1616,"session_id":"f532b587-630d-41e1-9bf2-56ed00c5bd35"} {"msg":"24749021-7c2f-4c30-b784-547e1c3475c8 is now active with session: f532b587-630d-41e1-9bf2-56ed00c5bd35","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:42.14921318Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1616,"session_id":"f532b587-630d-41e1-9bf2-56ed00c5bd35"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:42.149221869Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1616,"session_id":"f532b587-630d-41e1-9bf2-56ed00c5bd35"} {"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:42.149230988Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1616} {"msg":"Activated sub_volume 24749021-7c2f-4c30-b784-547e1c3475c8","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:42.149239106Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1616} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:42.149247075Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1616,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:42.149566398Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1616,"task":"crutest"} 01/30 Write block 6 len 24576 data: 9 8 9 7 6 5 02/30 Read block 258 len 40960 03/30 Read block 357 len 4096 04/30 Flush 05/30 Write block 315 len 8192 data: 12 12 06/30 Write block 457 len 20480 data: 12 8 9 8 8 07/30 Read block 339 len 8192 08/30 Read block 296 len 16384 09/30 Read block 586 len 40960 10/30 Write block 98 len 28672 data: 11 11 16 17 11 13 10 11/30 Read block 550 len 24576 12/30 Write block 536 len 28672 data: 11 8 8 7 6 8 8 13/30 Write block 97 len 12288 data: 11 12 12 14/30 Read block 384 len 16384 15/30 Write block 235 len 28672 data: 8 8 9 9 7 8 7 16/30 Write block 49 len 20480 data: 11 11 9 7 6 17/30 Read block 339 len 36864 18/30 Read block 321 len 40960 19/30 Flush 20/30 Write block 45 len 40960 data: 14 15 14 13 12 12 10 8 7 10 21/30 Write block 62 len 20480 data: 8 7 7 6 7 22/30 Write block 485 len 8192 data: 8 6 23/30 Read block 444 len 40960 24/30 Read block 549 len 8192 25/30 Read block 205 len 16384 26/30 Read block 310 len 8192 27/30 Read block 43 len 12288 28/30 Write block 484 len 36864 data: 7 9 7 9 12 9 8 5 6 29/30 Read block 357 len 8192 30/30 Read block 262 len 20480 ---------------------------------------------------------------- Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Write 2 Done Done Done false 1005 Acked Write 5 Done Done Done false 1006 Acked Read 2 Done Done Done false 1007 Acked Read 4 Done Done Sent false 1008 Acked Read 10 Done Done Sent false 1009 Acked Write 7 Done Done Sent false 1010 Acked Read 6 Done Done Sent false 1011 Acked Write 7 Done Done Sent false 1012 Acked Write 3 Done Done Sent false 1013 Acked Read 4 Done Done Sent false 1014 Acked Write 7 Done Done Sent false 1015 Acked Write 5 Done Done Sent false 1016 Acked Read 9 Done Done Sent false 1017 Acked Read 10 Done Done Sent false 1018 Acked Flush 0 Done Done Sent false 1019 Acked Write 10 Done Done Sent false 1020 Acked Write 5 Done Done Sent false 1021 Acked Write 2 Done Done Sent false 1022 Acked Read 10 Done Done Sent false 1023 Acked Read 2 Done Done Sent false 1024 Acked Read 4 Done Done Sent false 1025 Acked Read 2 Done Done Sent false 1026 Acked Read 3 Done Done Sent false 1027 Acked Write 9 Done Done Sent false 1028 Acked Read 2 Done Done Sent false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 23 24 Done 26 25 3 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) Some(JobId(1003)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:57:42.158893112Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1616} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:42.159080934Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1616,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 120 120 120 210 210 210 F F F 1 020-039 110 110 110 189 189 189 F F F 2 040-059 120 120 118 211 211 207 T T F <--- 3 060-079 116 116 116 205 205 205 T T F <--- 4 080-099 120 120 119 211 211 209 F F F <--- 5 100-119 120 120 119 211 211 209 F F F <--- 6 120-139 119 119 119 209 209 209 F F F 7 140-159 119 119 119 209 209 209 F F F 8 160-179 119 119 119 209 209 209 F F F 9 180-199 112 112 112 194 194 194 F F F 10 200-219 112 112 112 194 194 194 F F F 11 220-239 120 120 110 211 211 189 F F F <--- 12 240-259 120 120 112 211 211 194 F F F <--- 13 260-279 118 118 118 207 207 207 F F F 14 280-299 116 116 116 204 204 204 F F F 15 300-319 120 120 118 211 211 208 F F T <--- 16 320-339 112 112 112 194 194 194 F F F 17 340-359 115 115 115 199 199 199 F F F 18 360-379 117 117 117 206 206 206 F F F 19 380-399 119 119 119 209 209 209 F F F 20 400-419 119 119 119 209 209 209 F F F 21 420-439 118 118 118 207 207 207 F F F 22 440-459 120 120 118 211 211 207 F F T <--- 23 460-479 120 120 118 211 211 208 F F T <--- 24 480-499 118 118 118 208 208 208 T T F <--- 25 500-519 116 116 116 202 202 202 F F F 26 520-539 120 120 116 211 211 205 F F F <--- 27 540-559 120 120 112 211 211 194 F F F <--- 28 560-579 112 112 112 194 194 194 F F F 29 580-599 115 115 115 199 199 199 F F F Max gen: 120, Max flush: 211 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 60 Verifying data now No change after verify Loop: 60 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 120 120 120 210 210 210 F F F 1 020-039 110 110 110 189 189 189 F F F 2 040-059 121 121 121 212 212 212 F F F 3 060-079 121 121 121 212 212 212 F F F 4 080-099 120 120 120 211 211 211 F F F 5 100-119 120 120 120 211 211 211 F F F 6 120-139 119 119 119 209 209 209 F F F 7 140-159 119 119 119 209 209 209 F F F 8 160-179 119 119 119 209 209 209 F F F 9 180-199 112 112 112 194 194 194 F F F 10 200-219 112 112 112 194 194 194 F F F 11 220-239 120 120 120 211 211 211 F F F 12 240-259 120 120 120 211 211 211 F F F 13 260-279 118 118 118 207 207 207 F F F 14 280-299 116 116 116 204 204 204 F F F 15 300-319 120 120 120 211 211 211 F F F 16 320-339 112 112 112 194 194 194 F F F 17 340-359 115 115 115 199 199 199 F F F 18 360-379 117 117 117 206 206 206 F F F 19 380-399 119 119 119 209 209 209 F F F 20 400-419 119 119 119 209 209 209 F F F 21 420-439 118 118 118 207 207 207 F F F 22 440-459 120 120 120 211 211 211 F F F 23 460-479 120 120 120 211 211 211 F F F 24 480-499 121 121 121 212 212 212 F F F 25 500-519 116 116 116 202 202 202 F F F 26 520-539 120 120 120 211 211 211 F F F 27 540-559 120 120 120 211 211 211 F F F 28 560-579 112 112 112 194 194 194 F F F 29 580-599 115 115 115 199 199 199 F F F Max gen: 121, Max flush: 212 Begin loop 61 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:46.402797665Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1624,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:46.410884992Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1624,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:46.411202495Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1624} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:46.411230391Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1624} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:46.41123915Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1624} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:46.411250768Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1624} {"msg":"Crucible 48b25c25-c7bb-4aa8-ad30-c3de8542cc51 has session id: 6fb49b7e-040c-411f-a5a2-40c7686ba698","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:46.411556843Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1624,"session_id":"6fb49b7e-040c-411f-a5a2-40c7686ba698"} {"msg":"Upstairs opts: Upstairs UUID: 48b25c25-c7bb-4aa8-ad30-c3de8542cc51, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:46.411647689Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1624,"session_id":"6fb49b7e-040c-411f-a5a2-40c7686ba698"} {"msg":"Crucible stats registered with UUID: 48b25c25-c7bb-4aa8-ad30-c3de8542cc51","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:46.411673206Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1624,"session_id":"6fb49b7e-040c-411f-a5a2-40c7686ba698"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:57:46.411683224Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1624,"session_id":"6fb49b7e-040c-411f-a5a2-40c7686ba698"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:46.411807056Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1624,"":"io task","client":"2","":"downstairs","session_id":"6fb49b7e-040c-411f-a5a2-40c7686ba698"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:46.411823483Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1624,"":"io task","client":"0","":"downstairs","session_id":"6fb49b7e-040c-411f-a5a2-40c7686ba698"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:46.411833612Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1624,"":"io task","client":"1","":"downstairs","session_id":"6fb49b7e-040c-411f-a5a2-40c7686ba698"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:46.41198483Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1624,"task":"crutest"} {"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:46.412000827Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1624} {"msg":"48b25c25-c7bb-4aa8-ad30-c3de8542cc51 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:46.412024444Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1624,"session_id":"6fb49b7e-040c-411f-a5a2-40c7686ba698"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:46.412033972Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1624,"client":"0","":"downstairs","session_id":"6fb49b7e-040c-411f-a5a2-40c7686ba698"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:46.412045921Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1624,"client":"1","":"downstairs","session_id":"6fb49b7e-040c-411f-a5a2-40c7686ba698"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:46.412055619Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1624,"client":"2","":"downstairs","session_id":"6fb49b7e-040c-411f-a5a2-40c7686ba698"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:46.412065308Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1624,"":"io task","client":"2","":"downstairs","session_id":"6fb49b7e-040c-411f-a5a2-40c7686ba698"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:46.412075316Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1624,"":"io task","client":"0","":"downstairs","session_id":"6fb49b7e-040c-411f-a5a2-40c7686ba698"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:46.412085425Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1624,"":"io task","client":"1","":"downstairs","session_id":"6fb49b7e-040c-411f-a5a2-40c7686ba698"} {"msg":"ds_connection connected from Ok(127.0.0.1:57307)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:46.412227354Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1624,"":"io task","client":"1","":"downstairs","session_id":"6fb49b7e-040c-411f-a5a2-40c7686ba698"} {"msg":"ds_connection connected from Ok(127.0.0.1:44681)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:46.412284545Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1624,"":"io task","client":"2","":"downstairs","session_id":"6fb49b7e-040c-411f-a5a2-40c7686ba698"} {"msg":"ds_connection connected from Ok(127.0.0.1:39389)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:46.412296424Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1624,"":"io task","client":"0","":"downstairs","session_id":"6fb49b7e-040c-411f-a5a2-40c7686ba698"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:46.412919592Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1624,"client":"2","":"downstairs","session_id":"6fb49b7e-040c-411f-a5a2-40c7686ba698"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:46.412944278Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1624,"client":"1","":"downstairs","session_id":"6fb49b7e-040c-411f-a5a2-40c7686ba698"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:46.41299964Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1624,"client":"0","":"downstairs","session_id":"6fb49b7e-040c-411f-a5a2-40c7686ba698"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:46.413260491Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1624,"session_id":"6fb49b7e-040c-411f-a5a2-40c7686ba698"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:46.413301885Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1624,"session_id":"6fb49b7e-040c-411f-a5a2-40c7686ba698"} {"msg":"[0]R flush_numbers[0..12]: [210, 189, 212, 212, 211, 211, 209, 209, 209, 194, 194, 211]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:46.413347369Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1624,"":"downstairs","":"downstairs","session_id":"6fb49b7e-040c-411f-a5a2-40c7686ba698"} {"msg":"[0]R generation[0..12]: [120, 110, 121, 121, 120, 120, 119, 119, 119, 112, 112, 120]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:46.413358317Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1624,"":"downstairs","":"downstairs","session_id":"6fb49b7e-040c-411f-a5a2-40c7686ba698"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:46.413371295Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1624,"":"downstairs","":"downstairs","session_id":"6fb49b7e-040c-411f-a5a2-40c7686ba698"} {"msg":"[1]R flush_numbers[0..12]: [210, 189, 212, 212, 211, 211, 209, 209, 209, 194, 194, 211]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:46.413381783Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1624,"":"downstairs","":"downstairs","session_id":"6fb49b7e-040c-411f-a5a2-40c7686ba698"} {"msg":"[1]R generation[0..12]: [120, 110, 121, 121, 120, 120, 119, 119, 119, 112, 112, 120]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:46.413392362Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1624,"":"downstairs","":"downstairs","session_id":"6fb49b7e-040c-411f-a5a2-40c7686ba698"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:46.41340336Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1624,"":"downstairs","":"downstairs","session_id":"6fb49b7e-040c-411f-a5a2-40c7686ba698"} {"msg":"[2]R flush_numbers[0..12]: [210, 189, 212, 212, 211, 211, 209, 209, 209, 194, 194, 211]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:46.413414109Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1624,"":"downstairs","":"downstairs","session_id":"6fb49b7e-040c-411f-a5a2-40c7686ba698"} {"msg":"[2]R generation[0..12]: [120, 110, 121, 121, 120, 120, 119, 119, 119, 112, 112, 120]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:46.413424647Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1624,"":"downstairs","":"downstairs","session_id":"6fb49b7e-040c-411f-a5a2-40c7686ba698"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:46.413435536Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1624,"":"downstairs","":"downstairs","session_id":"6fb49b7e-040c-411f-a5a2-40c7686ba698"} {"msg":"Max found gen is 122","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:46.413445124Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1624,"":"downstairs","":"downstairs","session_id":"6fb49b7e-040c-411f-a5a2-40c7686ba698"} {"msg":"Generation requested: 122 >= found:122","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:46.413454923Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1624,"":"downstairs","":"downstairs","session_id":"6fb49b7e-040c-411f-a5a2-40c7686ba698"} {"msg":"Next flush: 213","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:46.413464401Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1624,"":"downstairs","":"downstairs","session_id":"6fb49b7e-040c-411f-a5a2-40c7686ba698"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:46.41347365Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1624,"":"downstairs","":"downstairs","session_id":"6fb49b7e-040c-411f-a5a2-40c7686ba698"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:46.413483478Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1624,"session_id":"6fb49b7e-040c-411f-a5a2-40c7686ba698"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:46.413492197Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1624,"client":"0","":"downstairs","session_id":"6fb49b7e-040c-411f-a5a2-40c7686ba698"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:46.413502056Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1624,"client":"1","":"downstairs","session_id":"6fb49b7e-040c-411f-a5a2-40c7686ba698"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:46.413511624Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1624,"client":"2","":"downstairs","session_id":"6fb49b7e-040c-411f-a5a2-40c7686ba698"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:46.413522703Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1624,"session_id":"6fb49b7e-040c-411f-a5a2-40c7686ba698"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:46.413531781Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1624,"session_id":"6fb49b7e-040c-411f-a5a2-40c7686ba698"} {"msg":"48b25c25-c7bb-4aa8-ad30-c3de8542cc51 is now active with session: 6fb49b7e-040c-411f-a5a2-40c7686ba698","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:46.41354151Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1624,"session_id":"6fb49b7e-040c-411f-a5a2-40c7686ba698"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:46.413550299Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1624,"session_id":"6fb49b7e-040c-411f-a5a2-40c7686ba698"} {"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:46.413559447Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1624} {"msg":"Activated sub_volume 48b25c25-c7bb-4aa8-ad30-c3de8542cc51","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:46.413567396Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1624} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:46.413575575Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1624,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:46.413864762Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1624,"task":"crutest"} 01/30 Flush 02/30 Write block 551 len 20480 data: 10 13 12 13 11 03/30 Flush 04/30 Write block 547 len 24576 data: 8 8 10 10 11 14 05/30 Write block 104 len 40960 data: 11 11 8 9 9 6 7 11 11 10 06/30 Flush 07/30 Write block 501 len 32768 data: 8 10 10 8 8 6 7 5 08/30 Write block 228 len 12288 data: 7 8 9 09/30 Read block 347 len 20480 10/30 Flush 11/30 Write block 394 len 24576 data: 8 9 7 8 8 7 12/30 Write block 596 len 8192 data: 10 7 13/30 Write block 168 len 28672 data: 7 8 6 6 7 8 8 14/30 Write block 543 len 40960 data: 6 6 7 9 9 9 11 11 12 15 15/30 Read block 292 len 4096 16/30 Read block 521 len 12288 17/30 Read block 211 len 20480 18/30 Read block 277 len 24576 19/30 Write block 479 len 28672 data: 14 13 11 12 10 8 10 20/30 Read block 310 len 24576 21/30 Write block 139 len 36864 data: 10 11 12 12 11 14 13 11 12 22/30 Write block 549 len 20480 data: 12 12 13 16 13 23/30 Write block 151 len 28672 data: 10 10 11 9 4 7 9 24/30 Read block 562 len 12288 25/30 Write block 510 len 20480 data: 9 8 10 12 12 26/30 Read block 599 len 4096 27/30 Write block 261 len 8192 data: 9 8 28/30 Read block 204 len 36864 29/30 Write block 343 len 16384 data: 7 7 5 6 30/30 Write block 258 len 8192 data: 16 14 ---------------------------------------------------------------- Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Flush 0 Done Sent Done false 1001 Acked Write 5 Done Sent Done false 1002 Acked Flush 0 Done Sent Done false 1003 Acked Write 6 Done Sent Done false 1004 Acked Write 10 Done Sent Done false 1005 Acked Flush 0 Done Sent Done false 1006 Acked Write 8 Done Sent Done false 1007 Acked Write 3 Done Sent Done false 1008 Acked Read 5 Done Sent Done false 1009 Acked Flush 0 Done Sent Done false 1010 Acked Write 6 Done Sent Done false 1011 Acked Write 2 Done Sent Done false 1012 Acked Write 7 Done Sent Done false 1013 Acked Write 10 Done Sent Done false 1014 Acked Read 1 Done Sent Done false 1015 Acked Read 3 Done Sent Done false 1016 Acked Read 5 Done Sent Done false 1017 Acked Read 6 Done Sent Done false 1018 Acked Write 7 Done Sent Done false 1019 Acked Read 6 Done Sent Done false 1020 Acked Write 9 Done Sent Done false 1021 Acked Write 5 Done Sent Done false 1022 Acked Write 7 Done Sent Done false 1023 Acked Read 3 Done Sent Done false 1024 Acked Write 5 Done Sent Done false 1025 Acked Read 1 Done Sent Done false 1026 Acked Write 2 Done Sent Done false 1027 Acked Read 9 Done Sent Done false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 30 2 34 Done 28 0 28 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1009)) None Some(JobId(1009)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:57:46.422376036Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1624} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:46.422578597Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1624,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 120 120 120 210 210 210 F F F 1 020-039 110 110 110 189 189 189 F F F 2 040-059 121 121 121 212 212 212 F F F 3 060-079 121 121 121 212 212 212 F F F 4 080-099 120 120 120 211 211 211 F F F 5 100-119 122 120 122 215 211 215 F F F <--- 6 120-139 119 119 119 209 209 209 T F T <--- 7 140-159 119 119 119 209 209 209 T F T <--- 8 160-179 119 119 119 209 209 209 T F T <--- 9 180-199 112 112 112 194 194 194 F F F 10 200-219 112 112 112 194 194 194 F F F 11 220-239 122 120 122 216 211 216 F F F <--- 12 240-259 120 120 120 211 211 211 T F T <--- 13 260-279 118 118 118 207 207 207 T F T <--- 14 280-299 116 116 116 204 204 204 F F F 15 300-319 120 120 120 211 211 211 F F F 16 320-339 112 112 112 194 194 194 F F F 17 340-359 115 115 115 199 199 199 T F T <--- 18 360-379 117 117 117 206 206 206 F F F 19 380-399 119 119 119 209 209 209 T F T <--- 20 400-419 119 119 119 209 209 209 F F F 21 420-439 118 118 118 207 207 207 F F F 22 440-459 120 120 120 211 211 211 F F F 23 460-479 120 120 120 211 211 211 T F T <--- 24 480-499 121 121 121 212 212 212 T F T <--- 25 500-519 122 116 122 216 202 216 T F T <--- 26 520-539 120 120 120 211 211 211 F F F 27 540-559 122 120 122 215 211 215 T F T <--- 28 560-579 112 112 112 194 194 194 F F F 29 580-599 115 115 115 199 199 199 T F T <--- Max gen: 122, Max flush: 216 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 61 Verifying data now No change after verify Loop: 61 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 120 120 120 210 210 210 F F F 1 020-039 110 110 110 189 189 189 F F F 2 040-059 121 121 121 212 212 212 F F F 3 060-079 121 121 121 212 212 212 F F F 4 080-099 120 120 120 211 211 211 F F F 5 100-119 122 122 122 215 215 215 F F F 6 120-139 123 123 123 217 217 217 F F F 7 140-159 123 123 123 217 217 217 F F F 8 160-179 123 123 123 217 217 217 F F F 9 180-199 112 112 112 194 194 194 F F F 10 200-219 112 112 112 194 194 194 F F F 11 220-239 122 122 122 216 216 216 F F F 12 240-259 123 123 123 217 217 217 F F F 13 260-279 123 123 123 217 217 217 F F F 14 280-299 116 116 116 204 204 204 F F F 15 300-319 120 120 120 211 211 211 F F F 16 320-339 112 112 112 194 194 194 F F F 17 340-359 123 123 123 217 217 217 F F F 18 360-379 117 117 117 206 206 206 F F F 19 380-399 123 123 123 217 217 217 F F F 20 400-419 119 119 119 209 209 209 F F F 21 420-439 118 118 118 207 207 207 F F F 22 440-459 120 120 120 211 211 211 F F F 23 460-479 123 123 123 217 217 217 F F F 24 480-499 123 123 123 217 217 217 F F F 25 500-519 123 123 123 217 217 217 F F F 26 520-539 120 120 120 211 211 211 F F F 27 540-559 123 123 123 217 217 217 F F F 28 560-579 112 112 112 194 194 194 F F F 29 580-599 123 123 123 217 217 217 F F F Max gen: 123, Max flush: 217 Begin loop 62 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:50.678761838Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1632,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:50.686788724Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1632,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:50.687078281Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1632} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:50.687102377Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1632} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:50.687110826Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1632} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:50.687122325Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1632} {"msg":"Crucible ecf69344-374f-4781-b4f4-efe0850ee32b has session id: 0aa94a55-9e7c-4f39-abcd-c11adb1e7441","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:50.68748941Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1632,"session_id":"0aa94a55-9e7c-4f39-abcd-c11adb1e7441"} {"msg":"Upstairs opts: Upstairs UUID: ecf69344-374f-4781-b4f4-efe0850ee32b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:50.687510247Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1632,"session_id":"0aa94a55-9e7c-4f39-abcd-c11adb1e7441"} {"msg":"Crucible stats registered with UUID: ecf69344-374f-4781-b4f4-efe0850ee32b","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:50.687520296Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1632,"session_id":"0aa94a55-9e7c-4f39-abcd-c11adb1e7441"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:57:50.687529514Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1632,"session_id":"0aa94a55-9e7c-4f39-abcd-c11adb1e7441"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:50.687671803Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1632,"":"io task","client":"0","":"downstairs","session_id":"0aa94a55-9e7c-4f39-abcd-c11adb1e7441"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:50.687688221Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1632,"":"io task","client":"1","":"downstairs","session_id":"0aa94a55-9e7c-4f39-abcd-c11adb1e7441"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:50.68769812Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1632,"":"io task","client":"2","":"downstairs","session_id":"0aa94a55-9e7c-4f39-abcd-c11adb1e7441"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:50.687859266Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1632,"task":"crutest"} {"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:50.687874744Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1632} {"msg":"ecf69344-374f-4781-b4f4-efe0850ee32b active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:50.687887972Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1632,"session_id":"0aa94a55-9e7c-4f39-abcd-c11adb1e7441"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:50.6878968Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1632,"client":"0","":"downstairs","session_id":"0aa94a55-9e7c-4f39-abcd-c11adb1e7441"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:50.687906889Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1632,"client":"1","":"downstairs","session_id":"0aa94a55-9e7c-4f39-abcd-c11adb1e7441"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:50.687916607Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1632,"client":"2","":"downstairs","session_id":"0aa94a55-9e7c-4f39-abcd-c11adb1e7441"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:50.687926006Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1632,"":"io task","client":"2","":"downstairs","session_id":"0aa94a55-9e7c-4f39-abcd-c11adb1e7441"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:50.687936164Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1632,"":"io task","client":"0","":"downstairs","session_id":"0aa94a55-9e7c-4f39-abcd-c11adb1e7441"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:50.687946043Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1632,"":"io task","client":"1","":"downstairs","session_id":"0aa94a55-9e7c-4f39-abcd-c11adb1e7441"} {"msg":"ds_connection connected from Ok(127.0.0.1:33616)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:50.687973989Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1632,"":"io task","client":"0","":"downstairs","session_id":"0aa94a55-9e7c-4f39-abcd-c11adb1e7441"} {"msg":"ds_connection connected from Ok(127.0.0.1:46982)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:50.687984987Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1632,"":"io task","client":"2","":"downstairs","session_id":"0aa94a55-9e7c-4f39-abcd-c11adb1e7441"} {"msg":"ds_connection connected from Ok(127.0.0.1:46393)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:50.68803119Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1632,"":"io task","client":"1","":"downstairs","session_id":"0aa94a55-9e7c-4f39-abcd-c11adb1e7441"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:50.68864519Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1632,"client":"1","":"downstairs","session_id":"0aa94a55-9e7c-4f39-abcd-c11adb1e7441"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:50.688670516Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1632,"client":"0","":"downstairs","session_id":"0aa94a55-9e7c-4f39-abcd-c11adb1e7441"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:50.688738056Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1632,"client":"2","":"downstairs","session_id":"0aa94a55-9e7c-4f39-abcd-c11adb1e7441"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:50.688963973Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1632,"session_id":"0aa94a55-9e7c-4f39-abcd-c11adb1e7441"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:50.688974041Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1632,"session_id":"0aa94a55-9e7c-4f39-abcd-c11adb1e7441"} {"msg":"[0]R flush_numbers[0..12]: [210, 189, 212, 212, 211, 215, 217, 217, 217, 194, 194, 216]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:50.689011346Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1632,"":"downstairs","":"downstairs","session_id":"0aa94a55-9e7c-4f39-abcd-c11adb1e7441"} {"msg":"[0]R generation[0..12]: [120, 110, 121, 121, 120, 122, 123, 123, 123, 112, 112, 122]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:50.689022704Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1632,"":"downstairs","":"downstairs","session_id":"0aa94a55-9e7c-4f39-abcd-c11adb1e7441"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:50.689033663Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1632,"":"downstairs","":"downstairs","session_id":"0aa94a55-9e7c-4f39-abcd-c11adb1e7441"} {"msg":"[1]R flush_numbers[0..12]: [210, 189, 212, 212, 211, 215, 217, 217, 217, 194, 194, 216]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:50.689044451Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1632,"":"downstairs","":"downstairs","session_id":"0aa94a55-9e7c-4f39-abcd-c11adb1e7441"} {"msg":"[1]R generation[0..12]: [120, 110, 121, 121, 120, 122, 123, 123, 123, 112, 112, 122]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:50.689054989Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1632,"":"downstairs","":"downstairs","session_id":"0aa94a55-9e7c-4f39-abcd-c11adb1e7441"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:50.689065998Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1632,"":"downstairs","":"downstairs","session_id":"0aa94a55-9e7c-4f39-abcd-c11adb1e7441"} {"msg":"[2]R flush_numbers[0..12]: [210, 189, 212, 212, 211, 215, 217, 217, 217, 194, 194, 216]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:50.689079486Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1632,"":"downstairs","":"downstairs","session_id":"0aa94a55-9e7c-4f39-abcd-c11adb1e7441"} {"msg":"[2]R generation[0..12]: [120, 110, 121, 121, 120, 122, 123, 123, 123, 112, 112, 122]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:50.689090244Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1632,"":"downstairs","":"downstairs","session_id":"0aa94a55-9e7c-4f39-abcd-c11adb1e7441"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:50.689101303Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1632,"":"downstairs","":"downstairs","session_id":"0aa94a55-9e7c-4f39-abcd-c11adb1e7441"} {"msg":"Max found gen is 124","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:50.689110881Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1632,"":"downstairs","":"downstairs","session_id":"0aa94a55-9e7c-4f39-abcd-c11adb1e7441"} {"msg":"Generation requested: 124 >= found:124","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:50.68912076Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1632,"":"downstairs","":"downstairs","session_id":"0aa94a55-9e7c-4f39-abcd-c11adb1e7441"} {"msg":"Next flush: 218","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:50.689130228Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1632,"":"downstairs","":"downstairs","session_id":"0aa94a55-9e7c-4f39-abcd-c11adb1e7441"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:50.689139837Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1632,"":"downstairs","":"downstairs","session_id":"0aa94a55-9e7c-4f39-abcd-c11adb1e7441"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:50.689149765Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1632,"session_id":"0aa94a55-9e7c-4f39-abcd-c11adb1e7441"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:50.689158654Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1632,"client":"0","":"downstairs","session_id":"0aa94a55-9e7c-4f39-abcd-c11adb1e7441"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:50.689168603Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1632,"client":"1","":"downstairs","session_id":"0aa94a55-9e7c-4f39-abcd-c11adb1e7441"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:50.689178291Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1632,"client":"2","":"downstairs","session_id":"0aa94a55-9e7c-4f39-abcd-c11adb1e7441"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:50.68918851Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1632,"session_id":"0aa94a55-9e7c-4f39-abcd-c11adb1e7441"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:50.689197808Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1632,"session_id":"0aa94a55-9e7c-4f39-abcd-c11adb1e7441"} {"msg":"ecf69344-374f-4781-b4f4-efe0850ee32b is now active with session: 0aa94a55-9e7c-4f39-abcd-c11adb1e7441","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:50.689207777Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1632,"session_id":"0aa94a55-9e7c-4f39-abcd-c11adb1e7441"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:50.689216656Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1632,"session_id":"0aa94a55-9e7c-4f39-abcd-c11adb1e7441"} {"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:50.689225694Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1632} {"msg":"Activated sub_volume ecf69344-374f-4781-b4f4-efe0850ee32b","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:50.689235383Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1632} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:50.689243492Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1632,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:50.689543227Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1632,"task":"crutest"} 01/30 Flush 02/30 Write block 414 len 20480 data: 13 13 14 9 7 03/30 Write block 524 len 16384 data: 6 4 4 5 04/30 Read block 528 len 32768 05/30 Write block 315 len 12288 data: 13 13 9 06/30 Write block 362 len 40960 data: 6 11 12 15 15 13 13 11 12 10 07/30 Read block 510 len 8192 08/30 Read block 282 len 36864 09/30 Flush 10/30 Write block 360 len 12288 data: 11 7 7 11/30 Flush 12/30 Write block 80 len 36864 data: 6 4 4 4 7 6 8 8 13 13/30 Write block 224 len 40960 data: 8 6 5 6 8 9 10 8 10 8 14/30 Write block 441 len 36864 data: 9 7 7 9 9 9 9 12 13 15/30 Write block 383 len 12288 data: 10 7 8 16/30 Read block 153 len 4096 17/30 Read block 72 len 28672 18/30 Write block 289 len 28672 data: 9 10 8 8 7 5 5 19/30 Read block 2 len 24576 20/30 Write block 63 len 20480 data: 8 8 7 8 10 21/30 Read block 375 len 16384 22/30 Read block 144 len 28672 23/30 Read block 518 len 24576 24/30 Write block 528 len 40960 data: 6 5 6 8 9 10 10 10 12 9 25/30 Write block 21 len 32768 data: 12 13 9 12 12 13 10 11 26/30 Write block 489 len 4096 data: 10 27/30 Read block 463 len 32768 28/30 Write block 319 len 12288 data: 6 7 8 29/30 Read block 562 len 8192 30/30 Write block 354 len 4096 data: 11 ---------------------------------------------------------------- Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 5 Done Done Done false 1002 Acked Write 4 Done Done Done false 1003 Acked Read 8 Done Done Done false 1004 Acked Write 3 Done Done Done false 1005 Acked Write 10 Done Done Done false 1006 Acked Read 2 Done Done Sent false 1007 Acked Read 9 Done Done Sent false 1008 Acked Flush 0 Done Done Sent false 1009 Acked Write 3 Done Done Sent false 1010 Acked Flush 0 Done Done Sent false 1011 Acked Write 9 Done Done Sent false 1012 Acked Write 10 Done Done Sent false 1013 Acked Write 9 Done Done Sent false 1014 Acked Write 3 Done Done Sent false 1015 Acked Read 1 Done Done Sent false 1016 Acked Read 7 Done Done Sent false 1017 Acked Write 7 Done Done Sent false 1018 Acked Read 6 Done Done Sent false 1019 Acked Write 5 Done Done Sent false 1020 Acked Read 4 Done Done Sent false 1021 Acked Read 7 Done Done Sent false 1022 Acked Read 6 Done Done Sent false 1023 Acked Write 10 Done Done Sent false 1024 Acked Write 8 Done Done Sent false 1025 Acked Write 1 Done Done Sent false 1026 Acked Read 8 Done Done Sent false 1027 Acked Write 3 Done Done Sent false 1028 Acked Read 2 Done Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 24 27 Done 28 27 5 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1010)) Some(JobId(1010)) Some(JobId(1000)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:57:50.698221857Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1632} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:50.698414279Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1632,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 120 120 120 210 210 210 F F F 1 020-039 110 110 110 189 189 189 T T F <--- 2 040-059 121 121 121 212 212 212 F F F 3 060-079 121 121 121 212 212 212 T T F <--- 4 080-099 120 120 120 211 211 211 T T F <--- 5 100-119 122 122 122 215 215 215 F F F 6 120-139 123 123 123 217 217 217 F F F 7 140-159 123 123 123 217 217 217 F F F 8 160-179 123 123 123 217 217 217 F F F 9 180-199 112 112 112 194 194 194 F F F 10 200-219 112 112 112 194 194 194 F F F 11 220-239 122 122 122 216 216 216 T T F <--- 12 240-259 123 123 123 217 217 217 F F F 13 260-279 123 123 123 217 217 217 F F F 14 280-299 116 116 116 204 204 204 T T F <--- 15 300-319 124 124 120 219 219 211 T T T <--- 16 320-339 112 112 112 194 194 194 T T F <--- 17 340-359 123 123 123 217 217 217 T T F <--- 18 360-379 124 124 117 220 220 206 F F T <--- 19 380-399 123 123 123 217 217 217 T T F <--- 20 400-419 124 124 119 219 219 209 F F T <--- 21 420-439 118 118 118 207 207 207 F F F 22 440-459 120 120 120 211 211 211 T T F <--- 23 460-479 123 123 123 217 217 217 F F F 24 480-499 123 123 123 217 217 217 T T F <--- 25 500-519 123 123 123 217 217 217 F F F 26 520-539 124 124 120 219 219 211 T T T <--- 27 540-559 123 123 123 217 217 217 F F F 28 560-579 112 112 112 194 194 194 F F F 29 580-599 123 123 123 217 217 217 F F F Max gen: 124, Max flush: 220 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 62 Verifying data now No change after verify Loop: 62 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 120 120 120 210 210 210 F F F 1 020-039 125 125 125 221 221 221 F F F 2 040-059 121 121 121 212 212 212 F F F 3 060-079 125 125 125 221 221 221 F F F 4 080-099 125 125 125 221 221 221 F F F 5 100-119 122 122 122 215 215 215 F F F 6 120-139 123 123 123 217 217 217 F F F 7 140-159 123 123 123 217 217 217 F F F 8 160-179 123 123 123 217 217 217 F F F 9 180-199 112 112 112 194 194 194 F F F 10 200-219 112 112 112 194 194 194 F F F 11 220-239 125 125 125 221 221 221 F F F 12 240-259 123 123 123 217 217 217 F F F 13 260-279 123 123 123 217 217 217 F F F 14 280-299 125 125 125 221 221 221 F F F 15 300-319 125 125 125 221 221 221 F F F 16 320-339 125 125 125 221 221 221 F F F 17 340-359 125 125 125 221 221 221 F F F 18 360-379 124 124 124 220 220 220 F F F 19 380-399 125 125 125 221 221 221 F F F 20 400-419 124 124 124 219 219 219 F F F 21 420-439 118 118 118 207 207 207 F F F 22 440-459 125 125 125 221 221 221 F F F 23 460-479 123 123 123 217 217 217 F F F 24 480-499 125 125 125 221 221 221 F F F 25 500-519 123 123 123 217 217 217 F F F 26 520-539 125 125 125 221 221 221 F F F 27 540-559 123 123 123 217 217 217 F F F 28 560-579 112 112 112 194 194 194 F F F 29 580-599 123 123 123 217 217 217 F F F Max gen: 125, Max flush: 221 Begin loop 63 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:54.959456752Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1640,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:57:54.966649171Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1640,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:54.966704023Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1640,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:54.973528506Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1640,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:54.973792517Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1640} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:54.973818273Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1640} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:54.973826832Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1640} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:54.973838201Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1640} {"msg":"Crucible 6145ccab-7082-4334-a69d-b4358d85e047 has session id: 46ffed4f-f326-4593-bf78-1167d6ae4411","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:54.974194668Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1640,"session_id":"46ffed4f-f326-4593-bf78-1167d6ae4411"} {"msg":"Upstairs opts: Upstairs UUID: 6145ccab-7082-4334-a69d-b4358d85e047, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:54.97425096Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1640,"session_id":"46ffed4f-f326-4593-bf78-1167d6ae4411"} {"msg":"Crucible stats registered with UUID: 6145ccab-7082-4334-a69d-b4358d85e047","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:54.974261608Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1640,"session_id":"46ffed4f-f326-4593-bf78-1167d6ae4411"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:57:54.974273176Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1640,"session_id":"46ffed4f-f326-4593-bf78-1167d6ae4411"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:54.974411616Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1640,"":"io task","client":"2","":"downstairs","session_id":"46ffed4f-f326-4593-bf78-1167d6ae4411"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:54.974443761Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1640,"":"io task","client":"0","":"downstairs","session_id":"46ffed4f-f326-4593-bf78-1167d6ae4411"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:54.97445481Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1640,"":"io task","client":"1","":"downstairs","session_id":"46ffed4f-f326-4593-bf78-1167d6ae4411"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:54.974606287Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1640,"task":"crutest"} {"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:54.974620765Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1640} {"msg":"6145ccab-7082-4334-a69d-b4358d85e047 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:54.974633643Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1640,"session_id":"46ffed4f-f326-4593-bf78-1167d6ae4411"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:54.974642522Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1640,"client":"0","":"downstairs","session_id":"46ffed4f-f326-4593-bf78-1167d6ae4411"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:54.97465237Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1640,"client":"1","":"downstairs","session_id":"46ffed4f-f326-4593-bf78-1167d6ae4411"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:54.974662159Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1640,"client":"2","":"downstairs","session_id":"46ffed4f-f326-4593-bf78-1167d6ae4411"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:54.974671988Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1640,"":"io task","client":"2","":"downstairs","session_id":"46ffed4f-f326-4593-bf78-1167d6ae4411"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:54.974682256Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1640,"":"io task","client":"0","":"downstairs","session_id":"46ffed4f-f326-4593-bf78-1167d6ae4411"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:54.974692195Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1640,"":"io task","client":"1","":"downstairs","session_id":"46ffed4f-f326-4593-bf78-1167d6ae4411"} {"msg":"ds_connection connected from Ok(127.0.0.1:60354)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:54.974798759Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1640,"":"io task","client":"2","":"downstairs","session_id":"46ffed4f-f326-4593-bf78-1167d6ae4411"} {"msg":"ds_connection connected from Ok(127.0.0.1:43105)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:54.974824655Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1640,"":"io task","client":"0","":"downstairs","session_id":"46ffed4f-f326-4593-bf78-1167d6ae4411"} {"msg":"ds_connection connected from Ok(127.0.0.1:51590)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:54.974868569Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1640,"":"io task","client":"1","":"downstairs","session_id":"46ffed4f-f326-4593-bf78-1167d6ae4411"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:54.975543119Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1640,"client":"2","":"downstairs","session_id":"46ffed4f-f326-4593-bf78-1167d6ae4411"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:54.975577714Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1640,"client":"1","":"downstairs","session_id":"46ffed4f-f326-4593-bf78-1167d6ae4411"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:54.975700736Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1640,"client":"0","":"downstairs","session_id":"46ffed4f-f326-4593-bf78-1167d6ae4411"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:54.9758784Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1640,"session_id":"46ffed4f-f326-4593-bf78-1167d6ae4411"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:54.975895987Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1640,"session_id":"46ffed4f-f326-4593-bf78-1167d6ae4411"} {"msg":"[0]R flush_numbers[0..12]: [210, 221, 212, 221, 221, 215, 217, 217, 217, 194, 194, 221]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:54.976025448Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1640,"":"downstairs","":"downstairs","session_id":"46ffed4f-f326-4593-bf78-1167d6ae4411"} {"msg":"[0]R generation[0..12]: [120, 125, 121, 125, 125, 122, 123, 123, 123, 112, 112, 125]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:54.976037046Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1640,"":"downstairs","":"downstairs","session_id":"46ffed4f-f326-4593-bf78-1167d6ae4411"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:54.976048185Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1640,"":"downstairs","":"downstairs","session_id":"46ffed4f-f326-4593-bf78-1167d6ae4411"} {"msg":"[1]R flush_numbers[0..12]: [210, 221, 212, 221, 221, 215, 217, 217, 217, 194, 194, 221]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:54.976058933Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1640,"":"downstairs","":"downstairs","session_id":"46ffed4f-f326-4593-bf78-1167d6ae4411"} {"msg":"[1]R generation[0..12]: [120, 125, 121, 125, 125, 122, 123, 123, 123, 112, 112, 125]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:54.976069681Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1640,"":"downstairs","":"downstairs","session_id":"46ffed4f-f326-4593-bf78-1167d6ae4411"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:54.97608069Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1640,"":"downstairs","":"downstairs","session_id":"46ffed4f-f326-4593-bf78-1167d6ae4411"} {"msg":"[2]R flush_numbers[0..12]: [210, 221, 212, 221, 221, 215, 217, 217, 217, 194, 194, 221]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:54.976091508Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1640,"":"downstairs","":"downstairs","session_id":"46ffed4f-f326-4593-bf78-1167d6ae4411"} {"msg":"[2]R generation[0..12]: [120, 125, 121, 125, 125, 122, 123, 123, 123, 112, 112, 125]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:54.976102167Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1640,"":"downstairs","":"downstairs","session_id":"46ffed4f-f326-4593-bf78-1167d6ae4411"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:54.976113185Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1640,"":"downstairs","":"downstairs","session_id":"46ffed4f-f326-4593-bf78-1167d6ae4411"} {"msg":"Max found gen is 126","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:54.976125253Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1640,"":"downstairs","":"downstairs","session_id":"46ffed4f-f326-4593-bf78-1167d6ae4411"} {"msg":"Generation requested: 126 >= found:126","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:54.976135152Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1640,"":"downstairs","":"downstairs","session_id":"46ffed4f-f326-4593-bf78-1167d6ae4411"} {"msg":"Next flush: 222","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:54.97614447Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1640,"":"downstairs","":"downstairs","session_id":"46ffed4f-f326-4593-bf78-1167d6ae4411"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:54.976153899Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1640,"":"downstairs","":"downstairs","session_id":"46ffed4f-f326-4593-bf78-1167d6ae4411"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:54.976163787Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1640,"session_id":"46ffed4f-f326-4593-bf78-1167d6ae4411"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:54.976172536Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1640,"client":"0","":"downstairs","session_id":"46ffed4f-f326-4593-bf78-1167d6ae4411"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:54.976182215Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1640,"client":"1","":"downstairs","session_id":"46ffed4f-f326-4593-bf78-1167d6ae4411"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:54.976191963Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1640,"client":"2","":"downstairs","session_id":"46ffed4f-f326-4593-bf78-1167d6ae4411"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:54.976201842Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1640,"session_id":"46ffed4f-f326-4593-bf78-1167d6ae4411"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:54.97621102Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1640,"session_id":"46ffed4f-f326-4593-bf78-1167d6ae4411"} {"msg":"6145ccab-7082-4334-a69d-b4358d85e047 is now active with session: 46ffed4f-f326-4593-bf78-1167d6ae4411","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:54.976220939Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1640,"session_id":"46ffed4f-f326-4593-bf78-1167d6ae4411"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:54.976229658Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1640,"session_id":"46ffed4f-f326-4593-bf78-1167d6ae4411"} {"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:54.976238776Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1640} {"msg":"Activated sub_volume 6145ccab-7082-4334-a69d-b4358d85e047","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:54.976246695Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1640} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:54.976254894Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1640,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:54.976590335Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1640,"task":"crutest"} 01/30 Write block 175 len 24576 data: 9 11 11 11 9 8 02/30 Write block 62 len 4096 data: 9 03/30 Write block 532 len 12288 data: 10 11 11 04/30 Read block 500 len 36864 05/30 Write block 195 len 24576 data: 13 9 11 11 10 8 06/30 Write block 450 len 32768 data: 14 15 14 14 11 7 10 13 07/30 Write block 578 len 20480 data: 9 8 8 12 12 08/30 Read block 95 len 4096 09/30 Read block 168 len 24576 10/30 Flush 11/30 Write block 34 len 12288 data: 5 10 10 12/30 Read block 318 len 32768 13/30 Write block 191 len 28672 data: 16 16 16 15 14 10 12 14/30 Read block 194 len 16384 15/30 Write block 282 len 28672 data: 8 9 10 9 11 8 7 16/30 Read block 255 len 12288 17/30 Write block 345 len 4096 data: 6 18/30 Write block 510 len 36864 data: 10 9 11 13 13 12 11 10 13 19/30 Read block 464 len 8192 20/30 Read block 33 len 40960 21/30 Read block 166 len 8192 22/30 Write block 570 len 36864 data: 7 7 7 8 10 9 7 8 10 23/30 Read block 586 len 28672 24/30 Write block 268 len 32768 data: 9 9 12 14 13 11 12 13 25/30 Write block 532 len 20480 data: 11 12 12 11 13 26/30 Write block 403 len 4096 data: 6 27/30 Write block 282 len 40960 data: 9 10 11 10 12 9 8 10 11 9 28/30 Write block 441 len 16384 data: 10 8 8 10 29/30 Read block 144 len 32768 30/30 Write block 351 len 8192 data: 7 7 ---------------------------------------------------------------- Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Write 1 Done Done Done false 1002 Acked Write 3 Sent Done Done false 1003 Acked Read 9 Sent Done Done false 1004 Acked Write 6 Sent Done Done false 1005 Acked Write 8 Sent Done Done false 1006 Acked Write 5 Sent Done Done false 1007 Acked Read 1 Sent Done Done false 1008 Acked Read 6 Sent Done Done false 1009 Acked Flush 0 Sent Done Done false 1010 Acked Write 3 Sent Done Done false 1011 Acked Read 8 Sent Done Done false 1012 Acked Write 7 Sent Done Done false 1013 Acked Read 4 Sent Done Done false 1014 Acked Write 7 Sent Done Done false 1015 Acked Read 3 Sent Done Done false 1016 Acked Write 1 Sent Done Done false 1017 Acked Write 9 Sent Done Done false 1018 Acked Read 2 Sent Done Done false 1019 Acked Read 10 Sent Done Done false 1020 Acked Read 2 Sent Done Done false 1021 Acked Write 9 Sent Done Done false 1022 Acked Read 7 Sent Done Done false 1023 Acked Write 8 Sent Done Done false 1024 Acked Write 5 Sent Done Done false 1025 Acked Write 1 Sent Done Done false 1026 Acked Write 10 Sent Done Done false 1027 Acked Write 4 Sent Done Done false 1028 Acked Read 8 Sent Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 1 1 30 Done 2 29 29 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1009)) Some(JobId(1009)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:57:54.985018291Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1640} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:54.985203414Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1640,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 120 120 120 210 210 210 F F F 1 020-039 125 125 125 221 221 221 F T T <--- 2 040-059 121 121 121 212 212 212 F F F 3 060-079 125 126 126 221 222 222 T F F <--- 4 080-099 125 125 125 221 221 221 F F F 5 100-119 122 122 122 215 215 215 F F F 6 120-139 123 123 123 217 217 217 F F F 7 140-159 123 123 123 217 217 217 F F F 8 160-179 123 126 126 217 222 222 T F F <--- 9 180-199 112 126 126 194 222 222 T T T <--- 10 200-219 112 126 126 194 222 222 F F F <--- 11 220-239 125 125 125 221 221 221 F F F 12 240-259 123 123 123 217 217 217 F F F 13 260-279 123 123 123 217 217 217 F T T <--- 14 280-299 125 125 125 221 221 221 F T T <--- 15 300-319 125 125 125 221 221 221 F F F 16 320-339 125 125 125 221 221 221 F F F 17 340-359 125 125 125 221 221 221 F T T <--- 18 360-379 124 124 124 220 220 220 F F F 19 380-399 125 125 125 221 221 221 F F F 20 400-419 124 124 124 219 219 219 F T T <--- 21 420-439 118 118 118 207 207 207 F F F 22 440-459 125 126 126 221 222 222 F T T <--- 23 460-479 123 123 123 217 217 217 F F F 24 480-499 125 125 125 221 221 221 F F F 25 500-519 123 123 123 217 217 217 F T T <--- 26 520-539 125 126 126 221 222 222 F T T <--- 27 540-559 123 123 123 217 217 217 F F F 28 560-579 112 126 126 194 222 222 F T T <--- 29 580-599 123 126 126 217 222 222 F F F <--- Max gen: 126, Max flush: 222 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 63 Verifying data now No change after verify Loop: 63 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 120 120 120 210 210 210 F F F 1 020-039 127 127 127 223 223 223 F F F 2 040-059 121 121 121 212 212 212 F F F 3 060-079 126 126 126 222 222 222 F F F 4 080-099 125 125 125 221 221 221 F F F 5 100-119 122 122 122 215 215 215 F F F 6 120-139 123 123 123 217 217 217 F F F 7 140-159 123 123 123 217 217 217 F F F 8 160-179 126 126 126 222 222 222 F F F 9 180-199 127 127 127 223 223 223 F F F 10 200-219 126 126 126 222 222 222 F F F 11 220-239 125 125 125 221 221 221 F F F 12 240-259 123 123 123 217 217 217 F F F 13 260-279 127 127 127 223 223 223 F F F 14 280-299 127 127 127 223 223 223 F F F 15 300-319 125 125 125 221 221 221 F F F 16 320-339 125 125 125 221 221 221 F F F 17 340-359 127 127 127 223 223 223 F F F 18 360-379 124 124 124 220 220 220 F F F 19 380-399 125 125 125 221 221 221 F F F 20 400-419 127 127 127 223 223 223 F F F 21 420-439 118 118 118 207 207 207 F F F 22 440-459 127 127 127 223 223 223 F F F 23 460-479 123 123 123 217 217 217 F F F 24 480-499 125 125 125 221 221 221 F F F 25 500-519 127 127 127 223 223 223 F F F 26 520-539 127 127 127 223 223 223 F F F 27 540-559 123 123 123 217 217 217 F F F 28 560-579 127 127 127 223 223 223 F F F 29 580-599 126 126 126 222 222 222 F F F Max gen: 127, Max flush: 223 Begin loop 64 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:59.251154244Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1648,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:57:59.258224651Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1648,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:59.258281783Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1648,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:59.265015439Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1648,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:59.2652812Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1648} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:59.265306656Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1648} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:59.265315105Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1648} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:59.265326643Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1648} {"msg":"Crucible f5cbee79-1b4c-4a4c-8612-70f226658ecf has session id: 02ee7080-56ae-443a-8c40-b48db33f145e","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:59.265642996Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1648,"session_id":"02ee7080-56ae-443a-8c40-b48db33f145e"} {"msg":"Upstairs opts: Upstairs UUID: f5cbee79-1b4c-4a4c-8612-70f226658ecf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:59.265659584Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1648,"session_id":"02ee7080-56ae-443a-8c40-b48db33f145e"} {"msg":"Crucible stats registered with UUID: f5cbee79-1b4c-4a4c-8612-70f226658ecf","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:59.265669623Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1648,"session_id":"02ee7080-56ae-443a-8c40-b48db33f145e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:57:59.265678921Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1648,"session_id":"02ee7080-56ae-443a-8c40-b48db33f145e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:59.265827239Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1648,"":"io task","client":"0","":"downstairs","session_id":"02ee7080-56ae-443a-8c40-b48db33f145e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:59.265843357Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1648,"":"io task","client":"1","":"downstairs","session_id":"02ee7080-56ae-443a-8c40-b48db33f145e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:59.265855495Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1648,"":"io task","client":"2","":"downstairs","session_id":"02ee7080-56ae-443a-8c40-b48db33f145e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:59.266006513Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1648,"task":"crutest"} {"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:59.266021621Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1648} {"msg":"f5cbee79-1b4c-4a4c-8612-70f226658ecf active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:59.266054456Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1648,"session_id":"02ee7080-56ae-443a-8c40-b48db33f145e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:59.266063974Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1648,"client":"0","":"downstairs","session_id":"02ee7080-56ae-443a-8c40-b48db33f145e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:59.266073823Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1648,"client":"1","":"downstairs","session_id":"02ee7080-56ae-443a-8c40-b48db33f145e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:59.266083361Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1648,"client":"2","":"downstairs","session_id":"02ee7080-56ae-443a-8c40-b48db33f145e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:59.26609303Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1648,"":"io task","client":"2","":"downstairs","session_id":"02ee7080-56ae-443a-8c40-b48db33f145e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:59.266103229Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1648,"":"io task","client":"0","":"downstairs","session_id":"02ee7080-56ae-443a-8c40-b48db33f145e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:59.266113417Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1648,"":"io task","client":"1","":"downstairs","session_id":"02ee7080-56ae-443a-8c40-b48db33f145e"} {"msg":"ds_connection connected from Ok(127.0.0.1:62351)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:59.266194335Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1648,"":"io task","client":"2","":"downstairs","session_id":"02ee7080-56ae-443a-8c40-b48db33f145e"} {"msg":"ds_connection connected from Ok(127.0.0.1:53039)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:59.266205503Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1648,"":"io task","client":"1","":"downstairs","session_id":"02ee7080-56ae-443a-8c40-b48db33f145e"} {"msg":"ds_connection connected from Ok(127.0.0.1:59439)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:59.266215922Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1648,"":"io task","client":"0","":"downstairs","session_id":"02ee7080-56ae-443a-8c40-b48db33f145e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:59.266936266Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1648,"client":"1","":"downstairs","session_id":"02ee7080-56ae-443a-8c40-b48db33f145e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:59.266957363Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1648,"client":"2","":"downstairs","session_id":"02ee7080-56ae-443a-8c40-b48db33f145e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:59.266984089Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1648,"client":"0","":"downstairs","session_id":"02ee7080-56ae-443a-8c40-b48db33f145e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:59.267286814Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1648,"session_id":"02ee7080-56ae-443a-8c40-b48db33f145e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:59.267297602Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1648,"session_id":"02ee7080-56ae-443a-8c40-b48db33f145e"} {"msg":"[0]R flush_numbers[0..12]: [210, 223, 212, 222, 221, 215, 217, 217, 222, 223, 222, 221]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:59.26731667Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1648,"":"downstairs","":"downstairs","session_id":"02ee7080-56ae-443a-8c40-b48db33f145e"} {"msg":"[0]R generation[0..12]: [120, 127, 121, 126, 125, 122, 123, 123, 126, 127, 126, 125]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:59.267327478Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1648,"":"downstairs","":"downstairs","session_id":"02ee7080-56ae-443a-8c40-b48db33f145e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:59.267338726Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1648,"":"downstairs","":"downstairs","session_id":"02ee7080-56ae-443a-8c40-b48db33f145e"} {"msg":"[1]R flush_numbers[0..12]: [210, 223, 212, 222, 221, 215, 217, 217, 222, 223, 222, 221]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:59.267349445Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1648,"":"downstairs","":"downstairs","session_id":"02ee7080-56ae-443a-8c40-b48db33f145e"} {"msg":"[1]R generation[0..12]: [120, 127, 121, 126, 125, 122, 123, 123, 126, 127, 126, 125]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:59.267360133Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1648,"":"downstairs","":"downstairs","session_id":"02ee7080-56ae-443a-8c40-b48db33f145e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:59.267371061Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1648,"":"downstairs","":"downstairs","session_id":"02ee7080-56ae-443a-8c40-b48db33f145e"} {"msg":"[2]R flush_numbers[0..12]: [210, 223, 212, 222, 221, 215, 217, 217, 222, 223, 222, 221]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:59.26738161Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1648,"":"downstairs","":"downstairs","session_id":"02ee7080-56ae-443a-8c40-b48db33f145e"} {"msg":"[2]R generation[0..12]: [120, 127, 121, 126, 125, 122, 123, 123, 126, 127, 126, 125]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:59.267392268Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1648,"":"downstairs","":"downstairs","session_id":"02ee7080-56ae-443a-8c40-b48db33f145e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:59.267402967Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1648,"":"downstairs","":"downstairs","session_id":"02ee7080-56ae-443a-8c40-b48db33f145e"} {"msg":"Max found gen is 128","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:59.267412455Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1648,"":"downstairs","":"downstairs","session_id":"02ee7080-56ae-443a-8c40-b48db33f145e"} {"msg":"Generation requested: 128 >= found:128","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:59.267422344Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1648,"":"downstairs","":"downstairs","session_id":"02ee7080-56ae-443a-8c40-b48db33f145e"} {"msg":"Next flush: 224","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:59.267431753Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1648,"":"downstairs","":"downstairs","session_id":"02ee7080-56ae-443a-8c40-b48db33f145e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:59.267442761Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1648,"":"downstairs","":"downstairs","session_id":"02ee7080-56ae-443a-8c40-b48db33f145e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:59.267452759Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1648,"session_id":"02ee7080-56ae-443a-8c40-b48db33f145e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:59.267461668Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1648,"client":"0","":"downstairs","session_id":"02ee7080-56ae-443a-8c40-b48db33f145e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:59.267471797Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1648,"client":"1","":"downstairs","session_id":"02ee7080-56ae-443a-8c40-b48db33f145e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:59.267481765Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1648,"client":"2","":"downstairs","session_id":"02ee7080-56ae-443a-8c40-b48db33f145e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:59.267491804Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1648,"session_id":"02ee7080-56ae-443a-8c40-b48db33f145e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:59.267501032Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1648,"session_id":"02ee7080-56ae-443a-8c40-b48db33f145e"} {"msg":"f5cbee79-1b4c-4a4c-8612-70f226658ecf is now active with session: 02ee7080-56ae-443a-8c40-b48db33f145e","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:59.267510861Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1648,"session_id":"02ee7080-56ae-443a-8c40-b48db33f145e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:59.26751963Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1648,"session_id":"02ee7080-56ae-443a-8c40-b48db33f145e"} {"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:59.267528508Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1648} {"msg":"Activated sub_volume f5cbee79-1b4c-4a4c-8612-70f226658ecf","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:59.267536537Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1648} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:59.267544596Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1648,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:59.267884086Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1648,"task":"crutest"} 01/30 Write block 527 len 16384 data: 6 7 6 7 02/30 Write block 414 len 28672 data: 14 14 15 10 8 8 8 03/30 Write block 220 len 20480 data: 7 9 9 8 9 04/30 Flush 05/30 Write block 499 len 40960 data: 9 10 9 11 11 9 9 7 8 6 06/30 Read block 130 len 12288 07/30 Read block 270 len 28672 08/30 Write block 320 len 8192 data: 8 9 09/30 Read block 301 len 8192 10/30 Write block 351 len 8192 data: 8 8 11/30 Write block 251 len 32768 data: 8 8 10 12 11 13 15 17 12/30 Read block 355 len 8192 13/30 Read block 351 len 28672 14/30 Read block 54 len 16384 15/30 Flush 16/30 Write block 235 len 4096 data: 9 17/30 Read block 524 len 16384 18/30 Read block 335 len 32768 19/30 Read block 108 len 4096 20/30 Flush 21/30 Write block 557 len 8192 data: 11 10 22/30 Write block 89 len 8192 data: 12 12 23/30 Read block 305 len 40960 24/30 Read block 78 len 28672 25/30 Write block 451 len 20480 data: 16 15 15 12 8 26/30 Write block 377 len 32768 data: 9 7 7 7 8 11 11 8 27/30 Read block 32 len 12288 28/30 Read block 378 len 16384 29/30 Read block 64 len 28672 30/30 Read block 543 len 8192 ---------------------------------------------------------------- Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Write 10 Done Done Done false 1005 Acked Read 3 Done Done Done false 1006 Acked Read 7 Done Done Done false 1007 Acked Write 2 Sent Done Done false 1008 Acked Read 2 Sent Done Done false 1009 Acked Write 2 Sent Done Done false 1010 Acked Write 8 Sent Done Done false 1011 Acked Read 2 Sent Done Done false 1012 Acked Read 7 Sent Done Done false 1013 Acked Read 4 Sent Done Done false 1014 Acked Flush 0 Sent Done Done false 1015 Acked Write 1 Sent Done Done false 1016 Acked Read 4 Sent Done Done false 1017 Acked Read 8 Sent Done Done false 1018 Acked Read 1 Sent Done Done false 1019 Acked Flush 0 Sent Done Done false 1020 Acked Write 2 Sent Done Done false 1021 Acked Write 2 Sent Done Done false 1022 Acked Read 10 Sent Done Done false 1023 Acked Read 7 Sent Done Done false 1024 Acked Write 5 Sent Done Done false 1025 Acked Write 8 Sent Done Done false 1026 Acked Read 3 Sent Done Done false 1027 Acked Read 4 Sent Done Done false 1028 Acked Read 7 Sent Done Done false 1029 Acked Read 2 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 23 1 0 24 Done 3 25 26 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1003)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:57:59.276115501Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1648} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:57:59.276293295Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1648,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 120 120 120 210 210 210 F F F 1 020-039 127 127 127 223 223 223 F F F 2 040-059 121 121 121 212 212 212 F F F 3 060-079 126 126 126 222 222 222 F F F 4 080-099 125 125 125 221 221 221 F T T <--- 5 100-119 122 122 122 215 215 215 F F F 6 120-139 123 123 123 217 217 217 F F F 7 140-159 123 123 123 217 217 217 F F F 8 160-179 126 126 126 222 222 222 F F F 9 180-199 127 127 127 223 223 223 F F F 10 200-219 126 126 126 222 222 222 F F F 11 220-239 128 128 128 224 226 226 F F F <--- 12 240-259 123 128 128 217 225 225 F F F <--- 13 260-279 127 127 127 223 223 223 F F F 14 280-299 127 127 127 223 223 223 F F F 15 300-319 125 125 125 221 221 221 F F F 16 320-339 125 128 128 221 225 225 F F F <--- 17 340-359 127 128 128 223 225 225 F F F <--- 18 360-379 124 124 124 220 220 220 F T T <--- 19 380-399 125 125 125 221 221 221 F T T <--- 20 400-419 128 128 128 224 224 224 F F F 21 420-439 128 128 128 224 224 224 F F F 22 440-459 127 127 127 223 223 223 F T T <--- 23 460-479 123 123 123 217 217 217 F F F 24 480-499 125 128 128 221 225 225 T F F <--- 25 500-519 127 128 128 223 225 225 T F F <--- 26 520-539 128 128 128 224 224 224 F F F 27 540-559 123 123 123 217 217 217 F T T <--- 28 560-579 127 127 127 223 223 223 F F F 29 580-599 126 126 126 222 222 222 F F F Max gen: 128, Max flush: 226 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 64 Verifying data now No change after verify Loop: 64 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 120 120 120 210 210 210 F F F 1 020-039 127 127 127 223 223 223 F F F 2 040-059 121 121 121 212 212 212 F F F 3 060-079 126 126 126 222 222 222 F F F 4 080-099 129 129 129 227 227 227 F F F 5 100-119 122 122 122 215 215 215 F F F 6 120-139 123 123 123 217 217 217 F F F 7 140-159 123 123 123 217 217 217 F F F 8 160-179 126 126 126 222 222 222 F F F 9 180-199 127 127 127 223 223 223 F F F 10 200-219 126 126 126 222 222 222 F F F 11 220-239 128 128 128 226 226 226 F F F 12 240-259 128 128 128 225 225 225 F F F 13 260-279 127 127 127 223 223 223 F F F 14 280-299 127 127 127 223 223 223 F F F 15 300-319 125 125 125 221 221 221 F F F 16 320-339 128 128 128 225 225 225 F F F 17 340-359 128 128 128 225 225 225 F F F 18 360-379 129 129 129 227 227 227 F F F 19 380-399 129 129 129 227 227 227 F F F 20 400-419 128 128 128 224 224 224 F F F 21 420-439 128 128 128 224 224 224 F F F 22 440-459 129 129 129 227 227 227 F F F 23 460-479 123 123 123 217 217 217 F F F 24 480-499 128 128 128 225 225 225 F F F 25 500-519 128 128 128 225 225 225 F F F 26 520-539 128 128 128 224 224 224 F F F 27 540-559 129 129 129 227 227 227 F F F 28 560-579 127 127 127 223 223 223 F F F 29 580-599 126 126 126 222 222 222 F F F Max gen: 129, Max flush: 227 Begin loop 65 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:03.532324959Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1656,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:58:03.539390526Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1656,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:03.539440209Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1656,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:03.546236386Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1656,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:03.546518865Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1656} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:03.546544511Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1656} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:03.54655295Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1656} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:03.546564708Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1656} {"msg":"Crucible 0f03ae79-29a6-4788-a758-d3f97ec7b41b has session id: 3932f85e-bfe9-4890-8a2a-3e6fce9f2900","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:03.546896059Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1656,"session_id":"3932f85e-bfe9-4890-8a2a-3e6fce9f2900"} {"msg":"Upstairs opts: Upstairs UUID: 0f03ae79-29a6-4788-a758-d3f97ec7b41b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:03.546947232Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1656,"session_id":"3932f85e-bfe9-4890-8a2a-3e6fce9f2900"} {"msg":"Crucible stats registered with UUID: 0f03ae79-29a6-4788-a758-d3f97ec7b41b","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:03.54695809Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1656,"session_id":"3932f85e-bfe9-4890-8a2a-3e6fce9f2900"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:58:03.546967079Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1656,"session_id":"3932f85e-bfe9-4890-8a2a-3e6fce9f2900"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:03.547114657Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1656,"":"io task","client":"1","":"downstairs","session_id":"3932f85e-bfe9-4890-8a2a-3e6fce9f2900"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:03.547146592Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1656,"":"io task","client":"0","":"downstairs","session_id":"3932f85e-bfe9-4890-8a2a-3e6fce9f2900"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:03.547157811Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1656,"":"io task","client":"2","":"downstairs","session_id":"3932f85e-bfe9-4890-8a2a-3e6fce9f2900"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:03.547305299Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1656,"task":"crutest"} {"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:03.547321206Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1656} {"msg":"0f03ae79-29a6-4788-a758-d3f97ec7b41b active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:03.547332325Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1656,"session_id":"3932f85e-bfe9-4890-8a2a-3e6fce9f2900"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:03.547341303Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1656,"client":"0","":"downstairs","session_id":"3932f85e-bfe9-4890-8a2a-3e6fce9f2900"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:03.547352932Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1656,"client":"1","":"downstairs","session_id":"3932f85e-bfe9-4890-8a2a-3e6fce9f2900"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:03.54736289Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1656,"client":"2","":"downstairs","session_id":"3932f85e-bfe9-4890-8a2a-3e6fce9f2900"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:03.547372829Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1656,"":"io task","client":"2","":"downstairs","session_id":"3932f85e-bfe9-4890-8a2a-3e6fce9f2900"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:03.547383047Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1656,"":"io task","client":"0","":"downstairs","session_id":"3932f85e-bfe9-4890-8a2a-3e6fce9f2900"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:03.547392886Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1656,"":"io task","client":"1","":"downstairs","session_id":"3932f85e-bfe9-4890-8a2a-3e6fce9f2900"} {"msg":"ds_connection connected from Ok(127.0.0.1:59933)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:03.547487152Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1656,"":"io task","client":"2","":"downstairs","session_id":"3932f85e-bfe9-4890-8a2a-3e6fce9f2900"} {"msg":"ds_connection connected from Ok(127.0.0.1:33130)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:03.547513068Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1656,"":"io task","client":"1","":"downstairs","session_id":"3932f85e-bfe9-4890-8a2a-3e6fce9f2900"} {"msg":"ds_connection connected from Ok(127.0.0.1:41713)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:03.547524336Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1656,"":"io task","client":"0","":"downstairs","session_id":"3932f85e-bfe9-4890-8a2a-3e6fce9f2900"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:03.548164862Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1656,"client":"2","":"downstairs","session_id":"3932f85e-bfe9-4890-8a2a-3e6fce9f2900"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:03.54818045Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1656,"client":"1","":"downstairs","session_id":"3932f85e-bfe9-4890-8a2a-3e6fce9f2900"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:03.548191658Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1656,"client":"0","":"downstairs","session_id":"3932f85e-bfe9-4890-8a2a-3e6fce9f2900"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:03.548490444Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1656,"session_id":"3932f85e-bfe9-4890-8a2a-3e6fce9f2900"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:03.548501782Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1656,"session_id":"3932f85e-bfe9-4890-8a2a-3e6fce9f2900"} {"msg":"[0]R flush_numbers[0..12]: [210, 223, 212, 222, 227, 215, 217, 217, 222, 223, 222, 226]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:03.548536297Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1656,"":"downstairs","":"downstairs","session_id":"3932f85e-bfe9-4890-8a2a-3e6fce9f2900"} {"msg":"[0]R generation[0..12]: [120, 127, 121, 126, 129, 122, 123, 123, 126, 127, 126, 128]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:03.548547685Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1656,"":"downstairs","":"downstairs","session_id":"3932f85e-bfe9-4890-8a2a-3e6fce9f2900"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:03.548561413Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1656,"":"downstairs","":"downstairs","session_id":"3932f85e-bfe9-4890-8a2a-3e6fce9f2900"} {"msg":"[1]R flush_numbers[0..12]: [210, 223, 212, 222, 227, 215, 217, 217, 222, 223, 222, 226]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:03.548572222Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1656,"":"downstairs","":"downstairs","session_id":"3932f85e-bfe9-4890-8a2a-3e6fce9f2900"} {"msg":"[1]R generation[0..12]: [120, 127, 121, 126, 129, 122, 123, 123, 126, 127, 126, 128]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:03.54858271Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1656,"":"downstairs","":"downstairs","session_id":"3932f85e-bfe9-4890-8a2a-3e6fce9f2900"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:03.548593549Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1656,"":"downstairs","":"downstairs","session_id":"3932f85e-bfe9-4890-8a2a-3e6fce9f2900"} {"msg":"[2]R flush_numbers[0..12]: [210, 223, 212, 222, 227, 215, 217, 217, 222, 223, 222, 226]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:03.548604107Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1656,"":"downstairs","":"downstairs","session_id":"3932f85e-bfe9-4890-8a2a-3e6fce9f2900"} {"msg":"[2]R generation[0..12]: [120, 127, 121, 126, 129, 122, 123, 123, 126, 127, 126, 128]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:03.548614656Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1656,"":"downstairs","":"downstairs","session_id":"3932f85e-bfe9-4890-8a2a-3e6fce9f2900"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:03.548625524Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1656,"":"downstairs","":"downstairs","session_id":"3932f85e-bfe9-4890-8a2a-3e6fce9f2900"} {"msg":"Max found gen is 130","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:03.548634833Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1656,"":"downstairs","":"downstairs","session_id":"3932f85e-bfe9-4890-8a2a-3e6fce9f2900"} {"msg":"Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:03.548644751Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1656,"":"downstairs","":"downstairs","session_id":"3932f85e-bfe9-4890-8a2a-3e6fce9f2900"} {"msg":"Next flush: 228","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:03.54865408Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1656,"":"downstairs","":"downstairs","session_id":"3932f85e-bfe9-4890-8a2a-3e6fce9f2900"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:03.548663678Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1656,"":"downstairs","":"downstairs","session_id":"3932f85e-bfe9-4890-8a2a-3e6fce9f2900"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:03.548673367Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1656,"session_id":"3932f85e-bfe9-4890-8a2a-3e6fce9f2900"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:03.548682196Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1656,"client":"0","":"downstairs","session_id":"3932f85e-bfe9-4890-8a2a-3e6fce9f2900"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:03.548692034Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1656,"client":"1","":"downstairs","session_id":"3932f85e-bfe9-4890-8a2a-3e6fce9f2900"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:03.548701713Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1656,"client":"2","":"downstairs","session_id":"3932f85e-bfe9-4890-8a2a-3e6fce9f2900"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:03.548713331Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1656,"session_id":"3932f85e-bfe9-4890-8a2a-3e6fce9f2900"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:03.54872245Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1656,"session_id":"3932f85e-bfe9-4890-8a2a-3e6fce9f2900"} {"msg":"0f03ae79-29a6-4788-a758-d3f97ec7b41b is now active with session: 3932f85e-bfe9-4890-8a2a-3e6fce9f2900","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:03.548732238Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1656,"session_id":"3932f85e-bfe9-4890-8a2a-3e6fce9f2900"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:03.548741097Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1656,"session_id":"3932f85e-bfe9-4890-8a2a-3e6fce9f2900"} {"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:03.548750206Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1656} {"msg":"Activated sub_volume 0f03ae79-29a6-4788-a758-d3f97ec7b41b","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:03.548758194Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1656} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:03.548766403Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1656,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:03.549093725Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1656,"task":"crutest"} 01/30 Write block 381 len 16384 data: 9 12 12 9 02/30 Write block 39 len 20480 data: 16 15 14 13 14 03/30 Read block 275 len 4096 04/30 Read block 401 len 8192 05/30 Write block 432 len 32768 data: 9 10 10 10 9 10 11 9 06/30 Write block 180 len 4096 data: 9 07/30 Write block 316 len 4096 data: 14 08/30 Write block 195 len 20480 data: 15 11 13 12 11 09/30 Write block 42 len 8192 data: 14 15 10/30 Read block 296 len 20480 11/30 Read block 85 len 4096 12/30 Write block 138 len 16384 data: 10 11 12 13 13/30 Read block 534 len 40960 14/30 Read block 444 len 32768 15/30 Flush 16/30 Write block 149 len 4096 data: 10 17/30 Read block 83 len 40960 18/30 Read block 436 len 32768 19/30 Flush 20/30 Write block 236 len 36864 data: 9 10 10 8 9 8 8 8 10 21/30 Write block 46 len 24576 data: 16 15 14 13 13 11 22/30 Read block 360 len 24576 23/30 Read block 186 len 12288 24/30 Read block 31 len 8192 25/30 Write block 86 len 28672 data: 9 9 14 13 13 11 11 26/30 Write block 394 len 40960 data: 9 10 8 9 9 8 7 5 3 7 27/30 Write block 513 len 40960 data: 14 14 13 12 11 14 12 12 12 9 28/30 Write block 203 len 12288 data: 9 11 12 29/30 Read block 464 len 24576 30/30 Read block 233 len 16384 ---------------------------------------------------------------- Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Sent Done Done false 1001 Acked Write 5 Sent Done Done false 1002 Acked Read 1 Sent Done Done false 1003 Acked Read 2 Sent Done Done false 1004 Acked Write 8 Sent Done Done false 1005 Acked Write 1 Sent Done Done false 1006 Acked Write 1 Sent Done Done false 1007 Acked Write 5 Sent Done Done false 1008 Acked Write 2 Sent Done Done false 1009 Acked Read 5 Sent Done Done false 1010 Acked Read 1 Sent Done Done false 1011 Acked Write 4 Sent Done Done false 1012 Acked Read 10 Sent Done Done false 1013 Acked Read 8 Sent Done Done false 1014 Acked Flush 0 Sent Done Done false 1015 Acked Write 1 Sent Done Done false 1016 Acked Read 10 Sent Done Done false 1017 Acked Read 8 Sent Done Done false 1018 Acked Flush 0 Sent Done Done false 1019 Acked Write 9 Sent Done Done false 1020 Acked Write 6 Sent Done Done false 1021 Acked Read 6 Sent Done Done false 1022 Acked Read 3 Sent Done Done false 1023 Acked Read 2 Sent Done Done false 1024 Acked Write 7 Sent Done Done false 1025 Acked Write 10 Sent Done Done false 1026 Acked Write 10 Sent Done Done false 1027 Acked Write 3 Sent Done Done false 1028 Acked Read 6 Sent Done Done false 1029 Acked Read 4 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 1 0 31 Done 0 29 30 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:58:03.557146777Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1656} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:03.55732777Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1656,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 120 120 120 210 210 210 F F F 1 020-039 127 130 130 223 228 228 F F F <--- 2 040-059 121 130 130 212 228 228 F T T <--- 3 060-079 126 126 126 222 222 222 F F F 4 080-099 129 129 129 227 227 227 F T T <--- 5 100-119 122 122 122 215 215 215 F F F 6 120-139 123 130 130 217 228 228 F F F <--- 7 140-159 123 130 130 217 229 229 F F F <--- 8 160-179 126 126 126 222 222 222 F F F 9 180-199 127 130 130 223 228 228 F F F <--- 10 200-219 126 126 126 222 222 222 F T T <--- 11 220-239 128 128 128 226 226 226 F T T <--- 12 240-259 128 128 128 225 225 225 F T T <--- 13 260-279 127 127 127 223 223 223 F F F 14 280-299 127 127 127 223 223 223 F F F 15 300-319 125 130 130 221 228 228 F F F <--- 16 320-339 128 128 128 225 225 225 F F F 17 340-359 128 128 128 225 225 225 F F F 18 360-379 129 129 129 227 227 227 F F F 19 380-399 129 130 130 227 228 228 F T T <--- 20 400-419 128 128 128 224 224 224 F T T <--- 21 420-439 128 130 130 224 228 228 F F F <--- 22 440-459 129 129 129 227 227 227 F F F 23 460-479 123 123 123 217 217 217 F F F 24 480-499 128 128 128 225 225 225 F F F 25 500-519 128 128 128 225 225 225 F T T <--- 26 520-539 128 128 128 224 224 224 F T T <--- 27 540-559 129 129 129 227 227 227 F F F 28 560-579 127 127 127 223 223 223 F F F 29 580-599 126 126 126 222 222 222 F F F Max gen: 130, Max flush: 229 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 65 Verifying data now No change after verify Loop: 65 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 120 120 120 210 210 210 F F F 1 020-039 130 130 130 228 228 228 F F F 2 040-059 131 131 131 230 230 230 F F F 3 060-079 126 126 126 222 222 222 F F F 4 080-099 131 131 131 230 230 230 F F F 5 100-119 122 122 122 215 215 215 F F F 6 120-139 130 130 130 228 228 228 F F F 7 140-159 130 130 130 229 229 229 F F F 8 160-179 126 126 126 222 222 222 F F F 9 180-199 130 130 130 228 228 228 F F F 10 200-219 131 131 131 230 230 230 F F F 11 220-239 131 131 131 230 230 230 F F F 12 240-259 131 131 131 230 230 230 F F F 13 260-279 127 127 127 223 223 223 F F F 14 280-299 127 127 127 223 223 223 F F F 15 300-319 130 130 130 228 228 228 F F F 16 320-339 128 128 128 225 225 225 F F F 17 340-359 128 128 128 225 225 225 F F F 18 360-379 129 129 129 227 227 227 F F F 19 380-399 131 131 131 230 230 230 F F F 20 400-419 131 131 131 230 230 230 F F F 21 420-439 130 130 130 228 228 228 F F F 22 440-459 129 129 129 227 227 227 F F F 23 460-479 123 123 123 217 217 217 F F F 24 480-499 128 128 128 225 225 225 F F F 25 500-519 131 131 131 230 230 230 F F F 26 520-539 131 131 131 230 230 230 F F F 27 540-559 129 129 129 227 227 227 F F F 28 560-579 127 127 127 223 223 223 F F F 29 580-599 126 126 126 222 222 222 F F F Max gen: 131, Max flush: 230 Begin loop 66 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:07.816604735Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1664,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:58:07.823706027Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1664,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:07.823761408Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1664,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:07.830491296Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1664,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:07.830759076Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1664} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:07.830783902Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1664} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:07.830792441Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1664} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:07.830806369Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1664} {"msg":"Crucible a90d0dab-27ef-4e16-b477-325fe206086a has session id: 602e3ec4-3167-4339-bf02-23a0981c9673","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:07.831161717Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1664,"session_id":"602e3ec4-3167-4339-bf02-23a0981c9673"} {"msg":"Upstairs opts: Upstairs UUID: a90d0dab-27ef-4e16-b477-325fe206086a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:07.831192752Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1664,"session_id":"602e3ec4-3167-4339-bf02-23a0981c9673"} {"msg":"Crucible stats registered with UUID: a90d0dab-27ef-4e16-b477-325fe206086a","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:07.83120404Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1664,"session_id":"602e3ec4-3167-4339-bf02-23a0981c9673"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:58:07.831213569Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1664,"session_id":"602e3ec4-3167-4339-bf02-23a0981c9673"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:07.831351029Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1664,"":"io task","client":"1","":"downstairs","session_id":"602e3ec4-3167-4339-bf02-23a0981c9673"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:07.831383224Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1664,"":"io task","client":"0","":"downstairs","session_id":"602e3ec4-3167-4339-bf02-23a0981c9673"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:07.831394452Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1664,"":"io task","client":"2","":"downstairs","session_id":"602e3ec4-3167-4339-bf02-23a0981c9673"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:07.831537571Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1664,"task":"crutest"} {"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:07.831551129Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1664} {"msg":"a90d0dab-27ef-4e16-b477-325fe206086a active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:07.831583324Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1664,"session_id":"602e3ec4-3167-4339-bf02-23a0981c9673"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:07.831592753Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1664,"client":"0","":"downstairs","session_id":"602e3ec4-3167-4339-bf02-23a0981c9673"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:07.831602692Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1664,"client":"1","":"downstairs","session_id":"602e3ec4-3167-4339-bf02-23a0981c9673"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:07.83161237Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1664,"client":"2","":"downstairs","session_id":"602e3ec4-3167-4339-bf02-23a0981c9673"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:07.831622209Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1664,"":"io task","client":"2","":"downstairs","session_id":"602e3ec4-3167-4339-bf02-23a0981c9673"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:07.831632447Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1664,"":"io task","client":"0","":"downstairs","session_id":"602e3ec4-3167-4339-bf02-23a0981c9673"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:07.831644845Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1664,"":"io task","client":"1","":"downstairs","session_id":"602e3ec4-3167-4339-bf02-23a0981c9673"} {"msg":"ds_connection connected from Ok(127.0.0.1:49580)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:07.83168177Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1664,"":"io task","client":"2","":"downstairs","session_id":"602e3ec4-3167-4339-bf02-23a0981c9673"} {"msg":"ds_connection connected from Ok(127.0.0.1:42946)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:07.831692838Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1664,"":"io task","client":"1","":"downstairs","session_id":"602e3ec4-3167-4339-bf02-23a0981c9673"} {"msg":"ds_connection connected from Ok(127.0.0.1:41823)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:07.83181785Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1664,"":"io task","client":"0","":"downstairs","session_id":"602e3ec4-3167-4339-bf02-23a0981c9673"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:07.832337053Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1664,"client":"2","":"downstairs","session_id":"602e3ec4-3167-4339-bf02-23a0981c9673"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:07.832367789Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1664,"client":"1","":"downstairs","session_id":"602e3ec4-3167-4339-bf02-23a0981c9673"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:07.832522816Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1664,"client":"0","":"downstairs","session_id":"602e3ec4-3167-4339-bf02-23a0981c9673"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:07.832668094Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1664,"session_id":"602e3ec4-3167-4339-bf02-23a0981c9673"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:07.832707049Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1664,"session_id":"602e3ec4-3167-4339-bf02-23a0981c9673"} {"msg":"[0]R flush_numbers[0..12]: [210, 228, 230, 222, 230, 215, 228, 229, 222, 228, 230, 230]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:07.832828751Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1664,"":"downstairs","":"downstairs","session_id":"602e3ec4-3167-4339-bf02-23a0981c9673"} {"msg":"[0]R generation[0..12]: [120, 130, 131, 126, 131, 122, 130, 130, 126, 130, 131, 131]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:07.832840159Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1664,"":"downstairs","":"downstairs","session_id":"602e3ec4-3167-4339-bf02-23a0981c9673"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:07.832851177Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1664,"":"downstairs","":"downstairs","session_id":"602e3ec4-3167-4339-bf02-23a0981c9673"} {"msg":"[1]R flush_numbers[0..12]: [210, 228, 230, 222, 230, 215, 228, 229, 222, 228, 230, 230]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:07.832861606Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1664,"":"downstairs","":"downstairs","session_id":"602e3ec4-3167-4339-bf02-23a0981c9673"} {"msg":"[1]R generation[0..12]: [120, 130, 131, 126, 131, 122, 130, 130, 126, 130, 131, 131]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:07.832872124Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1664,"":"downstairs","":"downstairs","session_id":"602e3ec4-3167-4339-bf02-23a0981c9673"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:07.832885402Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1664,"":"downstairs","":"downstairs","session_id":"602e3ec4-3167-4339-bf02-23a0981c9673"} {"msg":"[2]R flush_numbers[0..12]: [210, 228, 230, 222, 230, 215, 228, 229, 222, 228, 230, 230]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:07.832896371Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1664,"":"downstairs","":"downstairs","session_id":"602e3ec4-3167-4339-bf02-23a0981c9673"} {"msg":"[2]R generation[0..12]: [120, 130, 131, 126, 131, 122, 130, 130, 126, 130, 131, 131]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:07.832906999Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1664,"":"downstairs","":"downstairs","session_id":"602e3ec4-3167-4339-bf02-23a0981c9673"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:07.832918008Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1664,"":"downstairs","":"downstairs","session_id":"602e3ec4-3167-4339-bf02-23a0981c9673"} {"msg":"Max found gen is 132","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:07.832927316Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1664,"":"downstairs","":"downstairs","session_id":"602e3ec4-3167-4339-bf02-23a0981c9673"} {"msg":"Generation requested: 132 >= found:132","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:07.832937005Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1664,"":"downstairs","":"downstairs","session_id":"602e3ec4-3167-4339-bf02-23a0981c9673"} {"msg":"Next flush: 231","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:07.832946223Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1664,"":"downstairs","":"downstairs","session_id":"602e3ec4-3167-4339-bf02-23a0981c9673"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:07.832955782Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1664,"":"downstairs","":"downstairs","session_id":"602e3ec4-3167-4339-bf02-23a0981c9673"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:07.832965741Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1664,"session_id":"602e3ec4-3167-4339-bf02-23a0981c9673"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:07.832974639Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1664,"client":"0","":"downstairs","session_id":"602e3ec4-3167-4339-bf02-23a0981c9673"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:07.832984498Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1664,"client":"1","":"downstairs","session_id":"602e3ec4-3167-4339-bf02-23a0981c9673"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:07.832994226Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1664,"client":"2","":"downstairs","session_id":"602e3ec4-3167-4339-bf02-23a0981c9673"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:07.833004115Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1664,"session_id":"602e3ec4-3167-4339-bf02-23a0981c9673"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:07.833013303Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1664,"session_id":"602e3ec4-3167-4339-bf02-23a0981c9673"} {"msg":"a90d0dab-27ef-4e16-b477-325fe206086a is now active with session: 602e3ec4-3167-4339-bf02-23a0981c9673","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:07.833023122Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1664,"session_id":"602e3ec4-3167-4339-bf02-23a0981c9673"} {"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:07.8330339Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1664} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:07.833041879Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1664,"session_id":"602e3ec4-3167-4339-bf02-23a0981c9673"} {"msg":"Activated sub_volume a90d0dab-27ef-4e16-b477-325fe206086a","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:07.833051278Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1664} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:07.833059187Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1664,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:07.83337736Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1664,"task":"crutest"} 01/30 Write block 229 len 28672 data: 10 11 9 11 9 6 10 02/30 Write block 120 len 32768 data: 12 14 12 13 12 10 9 8 03/30 Read block 250 len 24576 04/30 Write block 284 len 40960 data: 12 11 13 10 9 11 12 10 9 8 05/30 Read block 542 len 24576 06/30 Write block 25 len 28672 data: 13 14 11 12 10 9 9 07/30 Write block 27 len 12288 data: 12 13 11 08/30 Read block 355 len 4096 09/30 Read block 29 len 36864 10/30 Read block 130 len 12288 11/30 Write block 587 len 24576 data: 13 10 9 7 9 11 12/30 Read block 518 len 28672 13/30 Read block 520 len 32768 14/30 Read block 433 len 4096 15/30 Write block 382 len 20480 data: 13 13 10 9 9 16/30 Read block 545 len 24576 17/30 Read block 30 len 24576 18/30 Write block 280 len 20480 data: 11 9 10 11 13 19/30 Read block 535 len 36864 20/30 Write block 92 len 8192 data: 12 12 21/30 Write block 159 len 8192 data: 8 8 22/30 Read block 273 len 28672 23/30 Write block 458 len 4096 data: 9 24/30 Flush 25/30 Write block 396 len 36864 data: 9 10 10 9 8 6 4 8 8 26/30 Write block 277 len 28672 data: 14 14 11 12 10 11 12 27/30 Read block 382 len 28672 28/30 Read block 297 len 12288 29/30 Read block 274 len 32768 30/30 Write block 394 len 16384 data: 10 11 10 11 ---------------------------------------------------------------- Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Write 8 Done Done Done false 1002 Acked Read 6 Sent Done Done false 1003 Acked Write 10 Sent Done Done false 1004 Acked Read 6 Sent Done Done false 1005 Acked Write 7 Sent Done Done false 1006 Acked Write 3 Sent Done Done false 1007 Acked Read 1 Sent Done Done false 1008 Acked Read 9 Sent Done Done false 1009 Acked Read 3 Sent Done Done false 1010 Acked Write 6 Sent Done Done false 1011 Acked Read 7 Sent Done Done false 1012 Acked Read 8 Sent Done Done false 1013 Acked Read 1 Sent Done Done false 1014 Acked Write 5 Sent Done Done false 1015 Acked Read 6 Sent Done Done false 1016 Acked Read 6 Sent Done Done false 1017 Acked Write 5 Sent Done Done false 1018 Acked Read 9 Sent Done Done false 1019 Acked Write 2 Sent Done Done false 1020 Acked Write 2 Sent Done Done false 1021 Acked Read 7 Sent Done Done false 1022 Acked Write 1 Sent Done Done false 1023 Acked Flush 0 Sent Done Done false 1024 Acked Write 9 Sent Done Done false 1025 Acked Write 7 Sent Done Done false 1026 Acked Read 7 Sent Done Done false 1027 Acked Read 3 Sent Done Done false 1028 Acked Read 8 Sent Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 1 1 30 Done 2 29 29 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:58:07.842261569Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1664} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:07.842448581Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1664,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 120 120 120 210 210 210 F F F 1 020-039 130 132 132 228 231 231 F F F <--- 2 040-059 131 131 131 230 230 230 F F F 3 060-079 126 126 126 222 222 222 F F F 4 080-099 131 132 132 230 231 231 F F F <--- 5 100-119 122 122 122 215 215 215 F F F 6 120-139 130 132 132 228 231 231 T F F <--- 7 140-159 130 132 132 229 231 231 F F F <--- 8 160-179 126 132 132 222 231 231 F F F <--- 9 180-199 130 130 130 228 228 228 F F F 10 200-219 131 131 131 230 230 230 F F F 11 220-239 131 132 132 230 231 231 T F F <--- 12 240-259 131 131 131 230 230 230 F F F 13 260-279 127 127 127 223 223 223 F T T <--- 14 280-299 127 132 132 223 231 231 F T T <--- 15 300-319 130 130 130 228 228 228 F F F 16 320-339 128 128 128 225 225 225 F F F 17 340-359 128 128 128 225 225 225 F F F 18 360-379 129 129 129 227 227 227 F F F 19 380-399 131 132 132 230 231 231 F T T <--- 20 400-419 131 131 131 230 230 230 F T T <--- 21 420-439 130 130 130 228 228 228 F F F 22 440-459 129 132 132 227 231 231 F F F <--- 23 460-479 123 123 123 217 217 217 F F F 24 480-499 128 128 128 225 225 225 F F F 25 500-519 131 131 131 230 230 230 F F F 26 520-539 131 131 131 230 230 230 F F F 27 540-559 129 129 129 227 227 227 F F F 28 560-579 127 127 127 223 223 223 F F F 29 580-599 126 132 132 222 231 231 F F F <--- Max gen: 132, Max flush: 231 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 66 Verifying data now No change after verify Loop: 66 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 120 120 120 210 210 210 F F F 1 020-039 132 132 132 231 231 231 F F F 2 040-059 131 131 131 230 230 230 F F F 3 060-079 126 126 126 222 222 222 F F F 4 080-099 132 132 132 231 231 231 F F F 5 100-119 122 122 122 215 215 215 F F F 6 120-139 132 132 132 231 231 231 F F F 7 140-159 132 132 132 231 231 231 F F F 8 160-179 132 132 132 231 231 231 F F F 9 180-199 130 130 130 228 228 228 F F F 10 200-219 131 131 131 230 230 230 F F F 11 220-239 132 132 132 231 231 231 F F F 12 240-259 131 131 131 230 230 230 F F F 13 260-279 133 133 133 232 232 232 F F F 14 280-299 133 133 133 232 232 232 F F F 15 300-319 130 130 130 228 228 228 F F F 16 320-339 128 128 128 225 225 225 F F F 17 340-359 128 128 128 225 225 225 F F F 18 360-379 129 129 129 227 227 227 F F F 19 380-399 133 133 133 232 232 232 F F F 20 400-419 133 133 133 232 232 232 F F F 21 420-439 130 130 130 228 228 228 F F F 22 440-459 132 132 132 231 231 231 F F F 23 460-479 123 123 123 217 217 217 F F F 24 480-499 128 128 128 225 225 225 F F F 25 500-519 131 131 131 230 230 230 F F F 26 520-539 131 131 131 230 230 230 F F F 27 540-559 129 129 129 227 227 227 F F F 28 560-579 127 127 127 223 223 223 F F F 29 580-599 132 132 132 231 231 231 F F F Max gen: 133, Max flush: 232 Begin loop 67 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:12.100801063Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1672,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:12.109003873Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1672,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:12.109306048Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1672} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:12.109331374Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1672} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:12.109340063Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1672} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:12.109351531Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1672} {"msg":"Crucible 612a2f45-3329-4235-a27e-9b61628dd84d has session id: 71ee40f8-3ba7-40b5-afc4-0899b029e5d2","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:12.109684722Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1672,"session_id":"71ee40f8-3ba7-40b5-afc4-0899b029e5d2"} {"msg":"Upstairs opts: Upstairs UUID: 612a2f45-3329-4235-a27e-9b61628dd84d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:12.109714128Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1672,"session_id":"71ee40f8-3ba7-40b5-afc4-0899b029e5d2"} {"msg":"Crucible stats registered with UUID: 612a2f45-3329-4235-a27e-9b61628dd84d","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:12.109725416Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1672,"session_id":"71ee40f8-3ba7-40b5-afc4-0899b029e5d2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:58:12.109737024Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1672,"session_id":"71ee40f8-3ba7-40b5-afc4-0899b029e5d2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:12.109867385Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1672,"":"io task","client":"2","":"downstairs","session_id":"71ee40f8-3ba7-40b5-afc4-0899b029e5d2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:12.109883883Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1672,"":"io task","client":"0","":"downstairs","session_id":"71ee40f8-3ba7-40b5-afc4-0899b029e5d2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:12.109894101Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1672,"":"io task","client":"1","":"downstairs","session_id":"71ee40f8-3ba7-40b5-afc4-0899b029e5d2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:12.110043329Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1672,"task":"crutest"} {"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:12.110070705Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1672} {"msg":"612a2f45-3329-4235-a27e-9b61628dd84d active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:12.11010366Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1672,"session_id":"71ee40f8-3ba7-40b5-afc4-0899b029e5d2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:12.110113219Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1672,"client":"0","":"downstairs","session_id":"71ee40f8-3ba7-40b5-afc4-0899b029e5d2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:12.110123097Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1672,"client":"1","":"downstairs","session_id":"71ee40f8-3ba7-40b5-afc4-0899b029e5d2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:12.110132716Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1672,"client":"2","":"downstairs","session_id":"71ee40f8-3ba7-40b5-afc4-0899b029e5d2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:12.110142365Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1672,"":"io task","client":"2","":"downstairs","session_id":"71ee40f8-3ba7-40b5-afc4-0899b029e5d2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:12.110152383Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1672,"":"io task","client":"0","":"downstairs","session_id":"71ee40f8-3ba7-40b5-afc4-0899b029e5d2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:12.110162322Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1672,"":"io task","client":"1","":"downstairs","session_id":"71ee40f8-3ba7-40b5-afc4-0899b029e5d2"} {"msg":"ds_connection connected from Ok(127.0.0.1:61478)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:12.110256348Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1672,"":"io task","client":"0","":"downstairs","session_id":"71ee40f8-3ba7-40b5-afc4-0899b029e5d2"} {"msg":"ds_connection connected from Ok(127.0.0.1:36379)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:12.110285573Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1672,"":"io task","client":"1","":"downstairs","session_id":"71ee40f8-3ba7-40b5-afc4-0899b029e5d2"} {"msg":"ds_connection connected from Ok(127.0.0.1:44931)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:12.110297092Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1672,"":"io task","client":"2","":"downstairs","session_id":"71ee40f8-3ba7-40b5-afc4-0899b029e5d2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:12.110934128Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1672,"client":"0","":"downstairs","session_id":"71ee40f8-3ba7-40b5-afc4-0899b029e5d2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:12.110946746Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1672,"client":"1","":"downstairs","session_id":"71ee40f8-3ba7-40b5-afc4-0899b029e5d2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:12.110957474Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1672,"client":"2","":"downstairs","session_id":"71ee40f8-3ba7-40b5-afc4-0899b029e5d2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:12.111280207Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1672,"session_id":"71ee40f8-3ba7-40b5-afc4-0899b029e5d2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:12.111289745Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1672,"session_id":"71ee40f8-3ba7-40b5-afc4-0899b029e5d2"} {"msg":"[0]R flush_numbers[0..12]: [210, 231, 230, 222, 231, 215, 231, 231, 231, 228, 230, 231]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:12.111332559Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1672,"":"downstairs","":"downstairs","session_id":"71ee40f8-3ba7-40b5-afc4-0899b029e5d2"} {"msg":"[0]R generation[0..12]: [120, 132, 131, 126, 132, 122, 132, 132, 132, 130, 131, 132]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:12.111343877Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1672,"":"downstairs","":"downstairs","session_id":"71ee40f8-3ba7-40b5-afc4-0899b029e5d2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:12.111354826Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1672,"":"downstairs","":"downstairs","session_id":"71ee40f8-3ba7-40b5-afc4-0899b029e5d2"} {"msg":"[1]R flush_numbers[0..12]: [210, 231, 230, 222, 231, 215, 231, 231, 231, 228, 230, 231]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:12.111365434Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1672,"":"downstairs","":"downstairs","session_id":"71ee40f8-3ba7-40b5-afc4-0899b029e5d2"} {"msg":"[1]R generation[0..12]: [120, 132, 131, 126, 132, 122, 132, 132, 132, 130, 131, 132]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:12.111376053Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1672,"":"downstairs","":"downstairs","session_id":"71ee40f8-3ba7-40b5-afc4-0899b029e5d2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:12.111387001Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1672,"":"downstairs","":"downstairs","session_id":"71ee40f8-3ba7-40b5-afc4-0899b029e5d2"} {"msg":"[2]R flush_numbers[0..12]: [210, 231, 230, 222, 231, 215, 231, 231, 231, 228, 230, 231]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:12.111397679Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1672,"":"downstairs","":"downstairs","session_id":"71ee40f8-3ba7-40b5-afc4-0899b029e5d2"} {"msg":"[2]R generation[0..12]: [120, 132, 131, 126, 132, 122, 132, 132, 132, 130, 131, 132]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:12.111408228Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1672,"":"downstairs","":"downstairs","session_id":"71ee40f8-3ba7-40b5-afc4-0899b029e5d2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:12.111419056Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1672,"":"downstairs","":"downstairs","session_id":"71ee40f8-3ba7-40b5-afc4-0899b029e5d2"} {"msg":"Max found gen is 134","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:12.111430215Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1672,"":"downstairs","":"downstairs","session_id":"71ee40f8-3ba7-40b5-afc4-0899b029e5d2"} {"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:12.111440033Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1672,"":"downstairs","":"downstairs","session_id":"71ee40f8-3ba7-40b5-afc4-0899b029e5d2"} {"msg":"Next flush: 233","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:12.111449412Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1672,"":"downstairs","":"downstairs","session_id":"71ee40f8-3ba7-40b5-afc4-0899b029e5d2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:12.11145886Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1672,"":"downstairs","":"downstairs","session_id":"71ee40f8-3ba7-40b5-afc4-0899b029e5d2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:12.111468909Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1672,"session_id":"71ee40f8-3ba7-40b5-afc4-0899b029e5d2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:12.111477738Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1672,"client":"0","":"downstairs","session_id":"71ee40f8-3ba7-40b5-afc4-0899b029e5d2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:12.111487616Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1672,"client":"1","":"downstairs","session_id":"71ee40f8-3ba7-40b5-afc4-0899b029e5d2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:12.111497595Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1672,"client":"2","":"downstairs","session_id":"71ee40f8-3ba7-40b5-afc4-0899b029e5d2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:12.111507723Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1672,"session_id":"71ee40f8-3ba7-40b5-afc4-0899b029e5d2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:12.111516842Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1672,"session_id":"71ee40f8-3ba7-40b5-afc4-0899b029e5d2"} {"msg":"612a2f45-3329-4235-a27e-9b61628dd84d is now active with session: 71ee40f8-3ba7-40b5-afc4-0899b029e5d2","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:12.11152669Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1672,"session_id":"71ee40f8-3ba7-40b5-afc4-0899b029e5d2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:12.111535389Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1672,"session_id":"71ee40f8-3ba7-40b5-afc4-0899b029e5d2"} {"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:12.111544398Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1672} {"msg":"Activated sub_volume 612a2f45-3329-4235-a27e-9b61628dd84d","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:12.111552337Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1672} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:12.111560405Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1672,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:12.111881768Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1672,"task":"crutest"} 01/30 Write block 285 len 24576 data: 12 14 11 10 12 13 02/30 Write block 363 len 12288 data: 12 13 16 03/30 Read block 337 len 24576 04/30 Write block 259 len 40960 data: 15 9 10 9 11 11 9 11 11 10 05/30 Read block 195 len 12288 06/30 Flush 07/30 Write block 466 len 8192 data: 15 13 08/30 Read block 297 len 8192 09/30 Read block 295 len 20480 10/30 Write block 144 len 40960 data: 15 14 12 13 12 11 10 11 11 12 11/30 Write block 590 len 8192 data: 8 10 12/30 Read block 75 len 16384 13/30 Write block 444 len 16384 data: 11 10 10 10 14/30 Write block 257 len 8192 data: 16 18 15/30 Write block 347 len 16384 data: 5 6 5 6 16/30 Read block 321 len 20480 17/30 Read block 354 len 28672 18/30 Flush 19/30 Flush 20/30 Write block 308 len 16384 data: 8 11 13 16 21/30 Read block 14 len 32768 22/30 Read block 59 len 4096 23/30 Flush 24/30 Write block 49 len 24576 data: 14 14 12 9 8 11 25/30 Write block 81 len 12288 data: 5 5 5 26/30 Read block 171 len 36864 27/30 Write block 172 len 28672 data: 8 9 9 10 12 12 12 28/30 Read block 215 len 16384 29/30 Write block 537 len 16384 data: 10 9 8 7 30/30 Write block 131 len 24576 data: 5 4 7 11 10 9 ---------------------------------------------------------------- Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Write 3 Done Sent Done false 1002 Acked Read 6 Done Sent Done false 1003 Acked Write 10 Done Sent Done false 1004 Acked Read 3 Done Sent Done false 1005 Acked Flush 0 Done Sent Done false 1006 Acked Write 2 Done Sent Done false 1007 Acked Read 2 Done Sent Done false 1008 Acked Read 5 Done Sent Done false 1009 Acked Write 10 Done Sent Done false 1010 Acked Write 2 Done Sent Done false 1011 Acked Read 4 Done Sent Done false 1012 Acked Write 4 Done Sent Done false 1013 Acked Write 2 Done Sent Done false 1014 Acked Write 4 Done Sent Done false 1015 Acked Read 5 Done Sent Done false 1016 Acked Read 7 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Write 4 Done Sent Done false 1020 Acked Read 8 Done Sent Done false 1021 Acked Read 1 Done Sent Done false 1022 Acked Flush 0 Done Sent Done false 1023 Acked Write 6 Done Sent Done false 1024 Acked Write 3 Done Sent Done false 1025 Acked Read 9 Done Sent Done false 1026 Acked Write 7 Done Sent Done false 1027 Acked Read 4 Done Sent Done false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 29 2 33 Done 28 1 28 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) None Some(JobId(1022)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:58:12.11986787Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1672} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:12.120058882Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1672,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 120 120 120 210 210 210 F F F 1 020-039 132 132 132 231 231 231 F F F 2 040-059 131 131 131 230 230 230 T F T <--- 3 060-079 126 126 126 222 222 222 F F F 4 080-099 132 132 132 231 231 231 T F T <--- 5 100-119 122 122 122 215 215 215 F F F 6 120-139 132 132 132 231 231 231 T F T <--- 7 140-159 134 132 134 234 231 234 F F F <--- 8 160-179 132 132 132 231 231 231 T F T <--- 9 180-199 130 130 130 228 228 228 F F F 10 200-219 131 131 131 230 230 230 F F F 11 220-239 132 132 132 231 231 231 F F F 12 240-259 134 131 134 234 230 234 F F F <--- 13 260-279 134 133 134 233 232 233 F F F <--- 14 280-299 134 133 134 233 232 233 F T F <--- 15 300-319 134 130 134 236 228 236 F F F <--- 16 320-339 128 128 128 225 225 225 F F F 17 340-359 134 128 134 234 225 234 F F F <--- 18 360-379 134 129 134 233 227 233 F F F <--- 19 380-399 133 133 133 232 232 232 F F F 20 400-419 133 133 133 232 232 232 F F F 21 420-439 130 130 130 228 228 228 F F F 22 440-459 134 132 134 234 231 234 F F F <--- 23 460-479 134 123 134 234 217 234 F F F <--- 24 480-499 128 128 128 225 225 225 F F F 25 500-519 131 131 131 230 230 230 F F F 26 520-539 131 131 131 230 230 230 T F T <--- 27 540-559 129 129 129 227 227 227 T F T <--- 28 560-579 127 127 127 223 223 223 F F F 29 580-599 134 132 134 234 231 234 F F F <--- Max gen: 134, Max flush: 236 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 67 Verifying data now No change after verify Loop: 67 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 120 120 120 210 210 210 F F F 1 020-039 132 132 132 231 231 231 F F F 2 040-059 135 135 135 237 237 237 F F F 3 060-079 126 126 126 222 222 222 F F F 4 080-099 135 135 135 237 237 237 F F F 5 100-119 122 122 122 215 215 215 F F F 6 120-139 135 135 135 237 237 237 F F F 7 140-159 134 134 134 234 234 234 F F F 8 160-179 135 135 135 237 237 237 F F F 9 180-199 130 130 130 228 228 228 F F F 10 200-219 131 131 131 230 230 230 F F F 11 220-239 132 132 132 231 231 231 F F F 12 240-259 134 134 134 234 234 234 F F F 13 260-279 134 134 134 233 233 233 F F F 14 280-299 134 134 134 233 233 233 F F F 15 300-319 134 134 134 236 236 236 F F F 16 320-339 128 128 128 225 225 225 F F F 17 340-359 134 134 134 234 234 234 F F F 18 360-379 134 134 134 233 233 233 F F F 19 380-399 133 133 133 232 232 232 F F F 20 400-419 133 133 133 232 232 232 F F F 21 420-439 130 130 130 228 228 228 F F F 22 440-459 134 134 134 234 234 234 F F F 23 460-479 134 134 134 234 234 234 F F F 24 480-499 128 128 128 225 225 225 F F F 25 500-519 131 131 131 230 230 230 F F F 26 520-539 135 135 135 237 237 237 F F F 27 540-559 135 135 135 237 237 237 F F F 28 560-579 127 127 127 223 223 223 F F F 29 580-599 134 134 134 234 234 234 F F F Max gen: 135, Max flush: 237 Begin loop 68 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:16.372904565Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1680,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:16.380986473Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1680,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:16.38127913Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1680} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:16.381303236Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1680} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:16.381311805Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1680} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:16.381323283Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1680} {"msg":"Crucible 74c4dcca-ca4c-446d-8eac-62119a0a6332 has session id: 63928b33-5c9a-48d2-be35-405cad53acc2","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:16.381674551Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1680,"session_id":"63928b33-5c9a-48d2-be35-405cad53acc2"} {"msg":"Upstairs opts: Upstairs UUID: 74c4dcca-ca4c-446d-8eac-62119a0a6332, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:16.381703257Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1680,"session_id":"63928b33-5c9a-48d2-be35-405cad53acc2"} {"msg":"Crucible stats registered with UUID: 74c4dcca-ca4c-446d-8eac-62119a0a6332","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:16.381714176Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1680,"session_id":"63928b33-5c9a-48d2-be35-405cad53acc2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:58:16.381723154Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1680,"session_id":"63928b33-5c9a-48d2-be35-405cad53acc2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:16.381845256Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1680,"":"io task","client":"2","":"downstairs","session_id":"63928b33-5c9a-48d2-be35-405cad53acc2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:16.381875722Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1680,"":"io task","client":"0","":"downstairs","session_id":"63928b33-5c9a-48d2-be35-405cad53acc2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:16.38188675Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1680,"":"io task","client":"1","":"downstairs","session_id":"63928b33-5c9a-48d2-be35-405cad53acc2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:16.382042497Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1680,"task":"crutest"} {"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:16.382056425Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1680} {"msg":"74c4dcca-ca4c-446d-8eac-62119a0a6332 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:16.382076132Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1680,"session_id":"63928b33-5c9a-48d2-be35-405cad53acc2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:16.382085181Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1680,"client":"0","":"downstairs","session_id":"63928b33-5c9a-48d2-be35-405cad53acc2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:16.382094829Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1680,"client":"1","":"downstairs","session_id":"63928b33-5c9a-48d2-be35-405cad53acc2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:16.382104538Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1680,"client":"2","":"downstairs","session_id":"63928b33-5c9a-48d2-be35-405cad53acc2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:16.382114147Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1680,"":"io task","client":"2","":"downstairs","session_id":"63928b33-5c9a-48d2-be35-405cad53acc2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:16.382124265Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1680,"":"io task","client":"0","":"downstairs","session_id":"63928b33-5c9a-48d2-be35-405cad53acc2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:16.382134204Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1680,"":"io task","client":"1","":"downstairs","session_id":"63928b33-5c9a-48d2-be35-405cad53acc2"} {"msg":"ds_connection connected from Ok(127.0.0.1:60410)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:16.382198704Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1680,"":"io task","client":"0","":"downstairs","session_id":"63928b33-5c9a-48d2-be35-405cad53acc2"} {"msg":"ds_connection connected from Ok(127.0.0.1:54194)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:16.382217291Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1680,"":"io task","client":"1","":"downstairs","session_id":"63928b33-5c9a-48d2-be35-405cad53acc2"} {"msg":"ds_connection connected from Ok(127.0.0.1:41556)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:16.38222782Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1680,"":"io task","client":"2","":"downstairs","session_id":"63928b33-5c9a-48d2-be35-405cad53acc2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:16.382880164Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1680,"client":"0","":"downstairs","session_id":"63928b33-5c9a-48d2-be35-405cad53acc2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:16.38290199Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1680,"client":"1","":"downstairs","session_id":"63928b33-5c9a-48d2-be35-405cad53acc2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:16.383026222Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1680,"client":"2","":"downstairs","session_id":"63928b33-5c9a-48d2-be35-405cad53acc2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:16.383197607Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1680,"session_id":"63928b33-5c9a-48d2-be35-405cad53acc2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:16.383216434Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1680,"session_id":"63928b33-5c9a-48d2-be35-405cad53acc2"} {"msg":"[0]R flush_numbers[0..12]: [210, 231, 237, 222, 237, 215, 237, 234, 237, 228, 230, 231]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:16.383305121Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1680,"":"downstairs","":"downstairs","session_id":"63928b33-5c9a-48d2-be35-405cad53acc2"} {"msg":"[0]R generation[0..12]: [120, 132, 135, 126, 135, 122, 135, 134, 135, 130, 131, 132]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:16.383316559Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1680,"":"downstairs","":"downstairs","session_id":"63928b33-5c9a-48d2-be35-405cad53acc2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:16.383327548Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1680,"":"downstairs","":"downstairs","session_id":"63928b33-5c9a-48d2-be35-405cad53acc2"} {"msg":"[1]R flush_numbers[0..12]: [210, 231, 237, 222, 237, 215, 237, 234, 237, 228, 230, 231]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:16.383338186Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1680,"":"downstairs","":"downstairs","session_id":"63928b33-5c9a-48d2-be35-405cad53acc2"} {"msg":"[1]R generation[0..12]: [120, 132, 135, 126, 135, 122, 135, 134, 135, 130, 131, 132]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:16.383348724Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1680,"":"downstairs","":"downstairs","session_id":"63928b33-5c9a-48d2-be35-405cad53acc2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:16.383359583Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1680,"":"downstairs","":"downstairs","session_id":"63928b33-5c9a-48d2-be35-405cad53acc2"} {"msg":"[2]R flush_numbers[0..12]: [210, 231, 237, 222, 237, 215, 237, 234, 237, 228, 230, 231]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:16.383370051Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1680,"":"downstairs","":"downstairs","session_id":"63928b33-5c9a-48d2-be35-405cad53acc2"} {"msg":"[2]R generation[0..12]: [120, 132, 135, 126, 135, 122, 135, 134, 135, 130, 131, 132]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:16.38338048Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1680,"":"downstairs","":"downstairs","session_id":"63928b33-5c9a-48d2-be35-405cad53acc2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:16.383391268Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1680,"":"downstairs","":"downstairs","session_id":"63928b33-5c9a-48d2-be35-405cad53acc2"} {"msg":"Max found gen is 136","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:16.383400917Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1680,"":"downstairs","":"downstairs","session_id":"63928b33-5c9a-48d2-be35-405cad53acc2"} {"msg":"Generation requested: 136 >= found:136","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:16.383410795Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1680,"":"downstairs","":"downstairs","session_id":"63928b33-5c9a-48d2-be35-405cad53acc2"} {"msg":"Next flush: 238","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:16.383420204Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1680,"":"downstairs","":"downstairs","session_id":"63928b33-5c9a-48d2-be35-405cad53acc2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:16.383429653Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1680,"":"downstairs","":"downstairs","session_id":"63928b33-5c9a-48d2-be35-405cad53acc2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:16.383439391Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1680,"session_id":"63928b33-5c9a-48d2-be35-405cad53acc2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:16.38344817Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1680,"client":"0","":"downstairs","session_id":"63928b33-5c9a-48d2-be35-405cad53acc2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:16.383459258Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1680,"client":"1","":"downstairs","session_id":"63928b33-5c9a-48d2-be35-405cad53acc2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:16.383469097Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1680,"client":"2","":"downstairs","session_id":"63928b33-5c9a-48d2-be35-405cad53acc2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:16.383478945Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1680,"session_id":"63928b33-5c9a-48d2-be35-405cad53acc2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:16.383488034Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1680,"session_id":"63928b33-5c9a-48d2-be35-405cad53acc2"} {"msg":"74c4dcca-ca4c-446d-8eac-62119a0a6332 is now active with session: 63928b33-5c9a-48d2-be35-405cad53acc2","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:16.383497982Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1680,"session_id":"63928b33-5c9a-48d2-be35-405cad53acc2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:16.383506781Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1680,"session_id":"63928b33-5c9a-48d2-be35-405cad53acc2"} {"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:16.3835159Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1680} {"msg":"Activated sub_volume 74c4dcca-ca4c-446d-8eac-62119a0a6332","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:16.383524379Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1680} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:16.383532437Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1680,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:16.383841972Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1680,"task":"crutest"} 01/30 Write block 46 len 28672 data: 17 16 15 15 15 13 10 02/30 Write block 116 len 36864 data: 10 11 11 12 13 15 13 14 13 03/30 Flush 04/30 Write block 591 len 8192 data: 11 12 05/30 Read block 381 len 8192 06/30 Write block 435 len 36864 data: 11 10 11 12 10 7 11 9 9 07/30 Read block 225 len 32768 08/30 Read block 156 len 16384 09/30 Write block 128 len 32768 data: 5 7 6 6 5 8 12 11 10/30 Read block 195 len 24576 11/30 Read block 211 len 40960 12/30 Read block 43 len 20480 13/30 Read block 114 len 16384 14/30 Read block 230 len 12288 15/30 Write block 180 len 8192 data: 10 8 16/30 Flush 17/30 Write block 252 len 12288 data: 9 11 13 18/30 Write block 212 len 16384 data: 10 10 10 10 19/30 Write block 326 len 40960 data: 14 11 9 9 10 9 8 7 7 7 20/30 Write block 431 len 12288 data: 9 10 11 21/30 Write block 339 len 40960 data: 11 10 9 8 8 8 7 7 6 7 22/30 Write block 28 len 8192 data: 14 12 23/30 Read block 449 len 16384 24/30 Read block 578 len 40960 25/30 Read block 94 len 36864 26/30 Write block 166 len 28672 data: 9 9 8 9 7 7 9 27/30 Write block 60 len 8192 data: 8 7 28/30 Write block 13 len 4096 data: 7 29/30 Read block 125 len 24576 30/30 Read block 393 len 20480 ---------------------------------------------------------------- Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Write 9 Done Done Done false 1002 Acked Flush 0 Done Done Sent false 1003 Acked Write 2 Done Done Sent false 1004 Acked Read 2 Done Done Sent false 1005 Acked Write 9 Done Done Sent false 1006 Acked Read 8 Done Done Sent false 1007 Acked Read 4 Done Done Sent false 1008 Acked Write 8 Done Done Sent false 1009 Acked Read 6 Done Done Sent false 1010 Acked Read 10 Done Done Sent false 1011 Acked Read 5 Done Done Sent false 1012 Acked Read 4 Done Done Sent false 1013 Acked Read 3 Done Done Sent false 1014 Acked Write 2 Done Done Sent false 1015 Acked Flush 0 Done Done Sent false 1016 Acked Write 3 Done Done Sent false 1017 Acked Write 4 Done Done Sent false 1018 Acked Write 10 Done Done Sent false 1019 Acked Write 3 Done Done Sent false 1020 Acked Write 10 Done Done Sent false 1021 Acked Write 2 Done Done Sent false 1022 Acked Read 4 Done Done Sent false 1023 Acked Read 10 Done Done Sent false 1024 Acked Read 9 Done Done Sent false 1025 Acked Write 7 Done Done Sent false 1026 Acked Write 2 Done Done Sent false 1027 Acked Write 1 Done Done Sent false 1028 Acked Read 6 Done Done Sent false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 28 29 Done 30 29 2 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) Some(JobId(1015)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:58:16.392194669Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1680} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:16.392377372Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1680,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 120 120 120 210 210 210 T T F <--- 1 020-039 132 132 132 231 231 231 T T F <--- 2 040-059 136 136 135 238 238 237 F F T <--- 3 060-079 126 126 126 222 222 222 T T F <--- 4 080-099 135 135 135 237 237 237 F F F 5 100-119 136 136 122 238 238 215 F F T <--- 6 120-139 136 136 135 239 239 237 F F T <--- 7 140-159 134 134 134 234 234 234 F F F 8 160-179 135 135 135 237 237 237 T T F <--- 9 180-199 136 136 130 239 239 228 F F F <--- 10 200-219 131 131 131 230 230 230 T T F <--- 11 220-239 132 132 132 231 231 231 F F F 12 240-259 134 134 134 234 234 234 T T F <--- 13 260-279 134 134 134 233 233 233 F F F 14 280-299 134 134 134 233 233 233 F F F 15 300-319 134 134 134 236 236 236 F F F 16 320-339 128 128 128 225 225 225 T T F <--- 17 340-359 134 134 134 234 234 234 T T F <--- 18 360-379 134 134 134 233 233 233 F F F 19 380-399 133 133 133 232 232 232 F F F 20 400-419 133 133 133 232 232 232 F F F 21 420-439 136 136 130 239 239 228 T T F <--- 22 440-459 136 136 134 239 239 234 F F F <--- 23 460-479 134 134 134 234 234 234 F F F 24 480-499 128 128 128 225 225 225 F F F 25 500-519 131 131 131 230 230 230 F F F 26 520-539 135 135 135 237 237 237 F F F 27 540-559 135 135 135 237 237 237 F F F 28 560-579 127 127 127 223 223 223 F F F 29 580-599 136 136 134 239 239 234 F F F <--- Max gen: 136, Max flush: 239 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 68 Verifying data now No change after verify Loop: 68 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 240 240 240 F F F 1 020-039 137 137 137 240 240 240 F F F 2 040-059 136 136 136 238 238 238 F F F 3 060-079 137 137 137 240 240 240 F F F 4 080-099 135 135 135 237 237 237 F F F 5 100-119 136 136 136 238 238 238 F F F 6 120-139 136 136 136 239 239 239 F F F 7 140-159 134 134 134 234 234 234 F F F 8 160-179 137 137 137 240 240 240 F F F 9 180-199 136 136 136 239 239 239 F F F 10 200-219 137 137 137 240 240 240 F F F 11 220-239 132 132 132 231 231 231 F F F 12 240-259 137 137 137 240 240 240 F F F 13 260-279 134 134 134 233 233 233 F F F 14 280-299 134 134 134 233 233 233 F F F 15 300-319 134 134 134 236 236 236 F F F 16 320-339 137 137 137 240 240 240 F F F 17 340-359 137 137 137 240 240 240 F F F 18 360-379 134 134 134 233 233 233 F F F 19 380-399 133 133 133 232 232 232 F F F 20 400-419 133 133 133 232 232 232 F F F 21 420-439 137 137 137 240 240 240 F F F 22 440-459 136 136 136 239 239 239 F F F 23 460-479 134 134 134 234 234 234 F F F 24 480-499 128 128 128 225 225 225 F F F 25 500-519 131 131 131 230 230 230 F F F 26 520-539 135 135 135 237 237 237 F F F 27 540-559 135 135 135 237 237 237 F F F 28 560-579 127 127 127 223 223 223 F F F 29 580-599 136 136 136 239 239 239 F F F Max gen: 137, Max flush: 240 Begin loop 69 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:20.653696466Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1688,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:20.661790682Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1688,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:20.662081639Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1688} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:20.662105915Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1688} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:20.662114724Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1688} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:20.662129482Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1688} {"msg":"Crucible 555b9b72-af65-4a22-9812-d497132afc27 has session id: 91d36270-c3e1-44d0-b78a-572719a7ed41","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:20.662471981Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1688,"session_id":"91d36270-c3e1-44d0-b78a-572719a7ed41"} {"msg":"Upstairs opts: Upstairs UUID: 555b9b72-af65-4a22-9812-d497132afc27, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:20.662501417Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1688,"session_id":"91d36270-c3e1-44d0-b78a-572719a7ed41"} {"msg":"Crucible stats registered with UUID: 555b9b72-af65-4a22-9812-d497132afc27","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:20.662512425Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1688,"session_id":"91d36270-c3e1-44d0-b78a-572719a7ed41"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:58:20.662521584Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1688,"session_id":"91d36270-c3e1-44d0-b78a-572719a7ed41"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:20.662649355Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1688,"":"io task","client":"1","":"downstairs","session_id":"91d36270-c3e1-44d0-b78a-572719a7ed41"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:20.662666123Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1688,"":"io task","client":"0","":"downstairs","session_id":"91d36270-c3e1-44d0-b78a-572719a7ed41"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:20.662676341Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1688,"":"io task","client":"2","":"downstairs","session_id":"91d36270-c3e1-44d0-b78a-572719a7ed41"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:20.662809652Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1688,"task":"crutest"} {"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:20.662828079Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1688} {"msg":"555b9b72-af65-4a22-9812-d497132afc27 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:20.662859714Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1688,"session_id":"91d36270-c3e1-44d0-b78a-572719a7ed41"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:20.662869203Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1688,"client":"0","":"downstairs","session_id":"91d36270-c3e1-44d0-b78a-572719a7ed41"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:20.662878981Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1688,"client":"1","":"downstairs","session_id":"91d36270-c3e1-44d0-b78a-572719a7ed41"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:20.66288864Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1688,"client":"2","":"downstairs","session_id":"91d36270-c3e1-44d0-b78a-572719a7ed41"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:20.662898598Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1688,"":"io task","client":"2","":"downstairs","session_id":"91d36270-c3e1-44d0-b78a-572719a7ed41"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:20.662910867Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1688,"":"io task","client":"0","":"downstairs","session_id":"91d36270-c3e1-44d0-b78a-572719a7ed41"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:20.662949821Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1688,"":"io task","client":"1","":"downstairs","session_id":"91d36270-c3e1-44d0-b78a-572719a7ed41"} {"msg":"ds_connection connected from Ok(127.0.0.1:43231)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:20.662974947Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1688,"":"io task","client":"1","":"downstairs","session_id":"91d36270-c3e1-44d0-b78a-572719a7ed41"} {"msg":"ds_connection connected from Ok(127.0.0.1:54756)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:20.662986056Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1688,"":"io task","client":"2","":"downstairs","session_id":"91d36270-c3e1-44d0-b78a-572719a7ed41"} {"msg":"ds_connection connected from Ok(127.0.0.1:52451)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:20.663010172Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1688,"":"io task","client":"0","":"downstairs","session_id":"91d36270-c3e1-44d0-b78a-572719a7ed41"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:20.663688432Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1688,"client":"2","":"downstairs","session_id":"91d36270-c3e1-44d0-b78a-572719a7ed41"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:20.663714928Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1688,"client":"0","":"downstairs","session_id":"91d36270-c3e1-44d0-b78a-572719a7ed41"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:20.663787067Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1688,"client":"1","":"downstairs","session_id":"91d36270-c3e1-44d0-b78a-572719a7ed41"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:20.664059597Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1688,"session_id":"91d36270-c3e1-44d0-b78a-572719a7ed41"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:20.664083794Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1688,"session_id":"91d36270-c3e1-44d0-b78a-572719a7ed41"} {"msg":"[0]R flush_numbers[0..12]: [240, 240, 238, 240, 237, 238, 239, 234, 240, 239, 240, 231]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:20.664094822Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1688,"":"downstairs","":"downstairs","session_id":"91d36270-c3e1-44d0-b78a-572719a7ed41"} {"msg":"[0]R generation[0..12]: [137, 137, 136, 137, 135, 136, 136, 134, 137, 136, 137, 132]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:20.66410609Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1688,"":"downstairs","":"downstairs","session_id":"91d36270-c3e1-44d0-b78a-572719a7ed41"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:20.664117579Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1688,"":"downstairs","":"downstairs","session_id":"91d36270-c3e1-44d0-b78a-572719a7ed41"} {"msg":"[1]R flush_numbers[0..12]: [240, 240, 238, 240, 237, 238, 239, 234, 240, 239, 240, 231]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:20.664128537Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1688,"":"downstairs","":"downstairs","session_id":"91d36270-c3e1-44d0-b78a-572719a7ed41"} {"msg":"[1]R generation[0..12]: [137, 137, 136, 137, 135, 136, 136, 134, 137, 136, 137, 132]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:20.664139115Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1688,"":"downstairs","":"downstairs","session_id":"91d36270-c3e1-44d0-b78a-572719a7ed41"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:20.664152653Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1688,"":"downstairs","":"downstairs","session_id":"91d36270-c3e1-44d0-b78a-572719a7ed41"} {"msg":"[2]R flush_numbers[0..12]: [240, 240, 238, 240, 237, 238, 239, 234, 240, 239, 240, 231]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:20.664163262Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1688,"":"downstairs","":"downstairs","session_id":"91d36270-c3e1-44d0-b78a-572719a7ed41"} {"msg":"[2]R generation[0..12]: [137, 137, 136, 137, 135, 136, 136, 134, 137, 136, 137, 132]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:20.66417384Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1688,"":"downstairs","":"downstairs","session_id":"91d36270-c3e1-44d0-b78a-572719a7ed41"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:20.664184789Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1688,"":"downstairs","":"downstairs","session_id":"91d36270-c3e1-44d0-b78a-572719a7ed41"} {"msg":"Max found gen is 138","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:20.664194367Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1688,"":"downstairs","":"downstairs","session_id":"91d36270-c3e1-44d0-b78a-572719a7ed41"} {"msg":"Generation requested: 138 >= found:138","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:20.664204266Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1688,"":"downstairs","":"downstairs","session_id":"91d36270-c3e1-44d0-b78a-572719a7ed41"} {"msg":"Next flush: 241","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:20.664213484Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1688,"":"downstairs","":"downstairs","session_id":"91d36270-c3e1-44d0-b78a-572719a7ed41"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:20.664222993Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1688,"":"downstairs","":"downstairs","session_id":"91d36270-c3e1-44d0-b78a-572719a7ed41"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:20.664232812Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1688,"session_id":"91d36270-c3e1-44d0-b78a-572719a7ed41"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:20.66424144Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1688,"client":"0","":"downstairs","session_id":"91d36270-c3e1-44d0-b78a-572719a7ed41"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:20.664251189Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1688,"client":"1","":"downstairs","session_id":"91d36270-c3e1-44d0-b78a-572719a7ed41"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:20.664260737Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1688,"client":"2","":"downstairs","session_id":"91d36270-c3e1-44d0-b78a-572719a7ed41"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:20.664270516Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1688,"session_id":"91d36270-c3e1-44d0-b78a-572719a7ed41"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:20.664279585Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1688,"session_id":"91d36270-c3e1-44d0-b78a-572719a7ed41"} {"msg":"555b9b72-af65-4a22-9812-d497132afc27 is now active with session: 91d36270-c3e1-44d0-b78a-572719a7ed41","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:20.664289603Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 1688,"session_id":"91d36270-c3e1-44d0-b78a-572719a7ed41"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:20.664305601Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1688,"session_id":"91d36270-c3e1-44d0-b78a-572719a7ed41"} {"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:20.66431473Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1688} {"msg":"Activated sub_volume 555b9b72-af65-4a22-9812-d497132afc27","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:20.664322508Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1688} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:20.664330537Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1688,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:20.664559173Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1688,"task":"crutest"} 01/30 Write block 587 len 28672 data: 14 11 10 9 12 13 12 02/30 Read block 358 len 24576 03/30 Read block 208 len 28672 04/30 Read block 547 len 4096 05/30 Write block 22 len 4096 data: 14 06/30 Read block 77 len 28672 07/30 Write block 228 len 16384 data: 9 11 12 10 08/30 Flush 09/30 Write block 320 len 40960 data: 9 10 10 12 13 14 15 12 10 10 10/30 Read block 45 len 12288 11/30 Read block 332 len 12288 12/30 Read block 493 len 36864 13/30 Read block 420 len 28672 14/30 Write block 268 len 28672 data: 11 10 13 15 14 12 13 15/30 Write block 285 len 20480 data: 13 15 12 11 13 16/30 Flush 17/30 Write block 462 len 36864 data: 8 8 10 11 16 14 14 14 12 18/30 Read block 252 len 40960 19/30 Read block 384 len 24576 20/30 Read block 55 len 24576 21/30 Write block 462 len 40960 data: 9 9 11 12 17 15 15 15 13 12 22/30 Read block 195 len 40960 23/30 Write block 270 len 32768 data: 14 16 15 13 14 14 11 15 24/30 Read block 364 len 28672 25/30 Read block 582 len 4096 26/30 Read block 429 len 12288 27/30 Write block 495 len 4096 data: 8 28/30 Write block 379 len 24576 data: 8 8 10 14 14 11 29/30 Write block 424 len 36864 data: 14 14 10 14 13 12 11 10 11 30/30 Write block 146 len 32768 data: 13 14 13 12 11 12 12 13 ---------------------------------------------------------------- Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Read 6 Done Done Done false 1002 Acked Read 7 Done Sent Done false 1003 Acked Read 1 Done Sent Done false 1004 Acked Write 1 Done Sent Done false 1005 Acked Read 7 Done Sent Done false 1006 Acked Write 4 Done Sent Done false 1007 Acked Flush 0 Done Sent Done false 1008 Acked Write 10 Done Sent Done false 1009 Acked Read 3 Done Sent Done false 1010 Acked Read 3 Done Sent Done false 1011 Acked Read 9 Done Sent Done false 1012 Acked Read 7 Done Sent Done false 1013 Acked Write 7 Done Sent Done false 1014 Acked Write 5 Done Sent Done false 1015 Acked Flush 0 Done Sent Done false 1016 Acked Write 9 Done Sent Done false 1017 Acked Read 10 Done Sent Done false 1018 Acked Read 6 Done Sent Done false 1019 Acked Read 6 Done Sent Done false 1020 Acked Write 10 Done Sent Done false 1021 Acked Read 10 Done Sent Done false 1022 Acked Write 8 Done Sent Done false 1023 Acked Read 7 Done Sent Done false 1024 Acked Read 1 Done Sent Done false 1025 Acked Read 3 Done Sent Done false 1026 Acked Write 1 Done Sent Done false 1027 Acked Write 6 Done Sent Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 28 2 32 Done 28 2 28 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) None Some(JobId(1015)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:58:20.673773744Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1688} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:20.673959757Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1688,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 240 240 240 F F F 1 020-039 138 137 138 241 240 241 F F F <--- 2 040-059 136 136 136 238 238 238 F F F 3 060-079 137 137 137 240 240 240 F F F 4 080-099 135 135 135 237 237 237 F F F 5 100-119 136 136 136 238 238 238 F F F 6 120-139 136 136 136 239 239 239 F F F 7 140-159 134 134 134 234 234 234 T F T <--- 8 160-179 137 137 137 240 240 240 F F F 9 180-199 136 136 136 239 239 239 F F F 10 200-219 137 137 137 240 240 240 F F F 11 220-239 138 132 138 241 231 241 F F F <--- 12 240-259 137 137 137 240 240 240 F F F 13 260-279 138 134 138 242 233 242 T F T <--- 14 280-299 138 134 138 242 233 242 F F F <--- 15 300-319 134 134 134 236 236 236 F F F 16 320-339 138 137 138 242 240 242 F F F <--- 17 340-359 137 137 137 240 240 240 F F F 18 360-379 134 134 134 233 233 233 T F T <--- 19 380-399 133 133 133 232 232 232 T F T <--- 20 400-419 133 133 133 232 232 232 F F F 21 420-439 137 137 137 240 240 240 T F T <--- 22 440-459 136 136 136 239 239 239 F F F 23 460-479 134 134 134 234 234 234 T F T <--- 24 480-499 128 128 128 225 225 225 T F T <--- 25 500-519 131 131 131 230 230 230 F F F 26 520-539 135 135 135 237 237 237 F F F 27 540-559 135 135 135 237 237 237 F F F 28 560-579 127 127 127 223 223 223 F F F 29 580-599 138 136 138 241 239 241 F T F <--- Max gen: 138, Max flush: 242 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 69 Verifying data now No change after verify Loop: 69 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 240 240 240 F F F 1 020-039 138 138 138 241 241 241 F F F 2 040-059 136 136 136 238 238 238 F F F 3 060-079 137 137 137 240 240 240 F F F 4 080-099 135 135 135 237 237 237 F F F 5 100-119 136 136 136 238 238 238 F F F 6 120-139 136 136 136 239 239 239 F F F 7 140-159 139 139 139 243 243 243 F F F 8 160-179 137 137 137 240 240 240 F F F 9 180-199 136 136 136 239 239 239 F F F 10 200-219 137 137 137 240 240 240 F F F 11 220-239 138 138 138 241 241 241 F F F 12 240-259 137 137 137 240 240 240 F F F 13 260-279 139 139 139 243 243 243 F F F 14 280-299 138 138 138 242 242 242 F F F 15 300-319 134 134 134 236 236 236 F F F 16 320-339 138 138 138 242 242 242 F F F 17 340-359 137 137 137 240 240 240 F F F 18 360-379 139 139 139 243 243 243 F F F 19 380-399 139 139 139 243 243 243 F F F 20 400-419 133 133 133 232 232 232 F F F 21 420-439 139 139 139 243 243 243 F F F 22 440-459 136 136 136 239 239 239 F F F 23 460-479 139 139 139 243 243 243 F F F 24 480-499 139 139 139 243 243 243 F F F 25 500-519 131 131 131 230 230 230 F F F 26 520-539 135 135 135 237 237 237 F F F 27 540-559 135 135 135 237 237 237 F F F 28 560-579 127 127 127 223 223 223 F F F 29 580-599 138 138 138 241 241 241 F F F Max gen: 139, Max flush: 243 Begin loop 70 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:24.921336317Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1696,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:58:24.928509028Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1696,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:24.928572689Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1696,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:24.935610321Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1696,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:24.93588842Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1696} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:24.935913896Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1696} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:24.935922685Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1696} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:24.935934043Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1696} {"msg":"Crucible 315d7a5c-4da7-40fe-bd86-8ac1de3fb736 has session id: 9661da8c-e6e4-4745-86c5-7e203cee988a","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:24.936260225Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1696,"session_id":"9661da8c-e6e4-4745-86c5-7e203cee988a"} {"msg":"Upstairs opts: Upstairs UUID: 315d7a5c-4da7-40fe-bd86-8ac1de3fb736, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:24.936283681Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1696,"session_id":"9661da8c-e6e4-4745-86c5-7e203cee988a"} {"msg":"Crucible stats registered with UUID: 315d7a5c-4da7-40fe-bd86-8ac1de3fb736","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:24.936295799Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1696,"session_id":"9661da8c-e6e4-4745-86c5-7e203cee988a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:58:24.936304908Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1696,"session_id":"9661da8c-e6e4-4745-86c5-7e203cee988a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:24.936449107Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1696,"":"io task","client":"2","":"downstairs","session_id":"9661da8c-e6e4-4745-86c5-7e203cee988a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:24.936466194Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1696,"":"io task","client":"0","":"downstairs","session_id":"9661da8c-e6e4-4745-86c5-7e203cee988a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:24.936476343Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1696,"":"io task","client":"1","":"downstairs","session_id":"9661da8c-e6e4-4745-86c5-7e203cee988a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:24.936651017Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1696,"task":"crutest"} {"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:24.936666495Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1696} {"msg":"315d7a5c-4da7-40fe-bd86-8ac1de3fb736 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:24.936682342Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1696,"session_id":"9661da8c-e6e4-4745-86c5-7e203cee988a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:24.936691281Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1696,"client":"0","":"downstairs","session_id":"9661da8c-e6e4-4745-86c5-7e203cee988a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:24.93670112Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1696,"client":"1","":"downstairs","session_id":"9661da8c-e6e4-4745-86c5-7e203cee988a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:24.936710718Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1696,"client":"2","":"downstairs","session_id":"9661da8c-e6e4-4745-86c5-7e203cee988a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:24.936720267Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1696,"":"io task","client":"2","":"downstairs","session_id":"9661da8c-e6e4-4745-86c5-7e203cee988a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:24.936730215Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1696,"":"io task","client":"0","":"downstairs","session_id":"9661da8c-e6e4-4745-86c5-7e203cee988a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:24.936740274Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1696,"":"io task","client":"1","":"downstairs","session_id":"9661da8c-e6e4-4745-86c5-7e203cee988a"} {"msg":"ds_connection connected from Ok(127.0.0.1:38910)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:24.93683133Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1696,"":"io task","client":"2","":"downstairs","session_id":"9661da8c-e6e4-4745-86c5-7e203cee988a"} {"msg":"ds_connection connected from Ok(127.0.0.1:49912)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:24.936947763Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1696,"":"io task","client":"0","":"downstairs","session_id":"9661da8c-e6e4-4745-86c5-7e203cee988a"} {"msg":"ds_connection connected from Ok(127.0.0.1:55214)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:24.936961041Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1696,"":"io task","client":"1","":"downstairs","session_id":"9661da8c-e6e4-4745-86c5-7e203cee988a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:24.93751246Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1696,"client":"2","":"downstairs","session_id":"9661da8c-e6e4-4745-86c5-7e203cee988a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:24.937530057Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1696,"client":"0","":"downstairs","session_id":"9661da8c-e6e4-4745-86c5-7e203cee988a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:24.937541166Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1696,"client":"1","":"downstairs","session_id":"9661da8c-e6e4-4745-86c5-7e203cee988a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:24.93785353Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1696,"session_id":"9661da8c-e6e4-4745-86c5-7e203cee988a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:24.937863258Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1696,"session_id":"9661da8c-e6e4-4745-86c5-7e203cee988a"} {"msg":"[0]R flush_numbers[0..12]: [240, 241, 238, 240, 237, 238, 239, 243, 240, 239, 240, 241]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:24.937897973Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1696,"":"downstairs","":"downstairs","session_id":"9661da8c-e6e4-4745-86c5-7e203cee988a"} {"msg":"[0]R generation[0..12]: [137, 138, 136, 137, 135, 136, 136, 139, 137, 136, 137, 138]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:24.937909381Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1696,"":"downstairs","":"downstairs","session_id":"9661da8c-e6e4-4745-86c5-7e203cee988a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:24.93792038Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1696,"":"downstairs","":"downstairs","session_id":"9661da8c-e6e4-4745-86c5-7e203cee988a"} {"msg":"[1]R flush_numbers[0..12]: [240, 241, 238, 240, 237, 238, 239, 243, 240, 239, 240, 241]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:24.937931058Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1696,"":"downstairs","":"downstairs","session_id":"9661da8c-e6e4-4745-86c5-7e203cee988a"} {"msg":"[1]R generation[0..12]: [137, 138, 136, 137, 135, 136, 136, 139, 137, 136, 137, 138]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:24.937941577Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1696,"":"downstairs","":"downstairs","session_id":"9661da8c-e6e4-4745-86c5-7e203cee988a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:24.937952235Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1696,"":"downstairs","":"downstairs","session_id":"9661da8c-e6e4-4745-86c5-7e203cee988a"} {"msg":"[2]R flush_numbers[0..12]: [240, 241, 238, 240, 237, 238, 239, 243, 240, 239, 240, 241]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:24.937962903Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1696,"":"downstairs","":"downstairs","session_id":"9661da8c-e6e4-4745-86c5-7e203cee988a"} {"msg":"[2]R generation[0..12]: [137, 138, 136, 137, 135, 136, 136, 139, 137, 136, 137, 138]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:24.937975362Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1696,"":"downstairs","":"downstairs","session_id":"9661da8c-e6e4-4745-86c5-7e203cee988a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:24.9379863Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1696,"":"downstairs","":"downstairs","session_id":"9661da8c-e6e4-4745-86c5-7e203cee988a"} {"msg":"Max found gen is 140","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:24.937996179Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1696,"":"downstairs","":"downstairs","session_id":"9661da8c-e6e4-4745-86c5-7e203cee988a"} {"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:24.938006067Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1696,"":"downstairs","":"downstairs","session_id":"9661da8c-e6e4-4745-86c5-7e203cee988a"} {"msg":"Next flush: 244","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:24.938015506Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1696,"":"downstairs","":"downstairs","session_id":"9661da8c-e6e4-4745-86c5-7e203cee988a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:24.938024884Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1696,"":"downstairs","":"downstairs","session_id":"9661da8c-e6e4-4745-86c5-7e203cee988a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:24.938034663Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1696,"session_id":"9661da8c-e6e4-4745-86c5-7e203cee988a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:24.938043402Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1696,"client":"0","":"downstairs","session_id":"9661da8c-e6e4-4745-86c5-7e203cee988a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:24.93805314Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1696,"client":"1","":"downstairs","session_id":"9661da8c-e6e4-4745-86c5-7e203cee988a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:24.938062829Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1696,"client":"2","":"downstairs","session_id":"9661da8c-e6e4-4745-86c5-7e203cee988a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:24.938072817Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1696,"session_id":"9661da8c-e6e4-4745-86c5-7e203cee988a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:24.938081946Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1696,"session_id":"9661da8c-e6e4-4745-86c5-7e203cee988a"} {"msg":"315d7a5c-4da7-40fe-bd86-8ac1de3fb736 is now active with session: 9661da8c-e6e4-4745-86c5-7e203cee988a","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:24.938091655Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1696,"session_id":"9661da8c-e6e4-4745-86c5-7e203cee988a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:24.938100433Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1696,"session_id":"9661da8c-e6e4-4745-86c5-7e203cee988a"} {"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:24.938109532Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1696} {"msg":"Activated sub_volume 315d7a5c-4da7-40fe-bd86-8ac1de3fb736","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:24.938117621Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1696} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:24.938125609Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1696,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:24.938495445Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1696,"task":"crutest"} 01/30 Write block 46 len 12288 data: 18 17 16 02/30 Read block 2 len 8192 03/30 Read block 77 len 24576 04/30 Read block 297 len 28672 05/30 Write block 508 len 16384 data: 7 5 11 10 06/30 Read block 554 len 28672 07/30 Read block 108 len 24576 08/30 Read block 52 len 12288 09/30 Read block 560 len 32768 10/30 Read block 237 len 16384 11/30 Read block 277 len 40960 12/30 Write block 254 len 12288 data: 14 12 14 13/30 Read block 224 len 20480 14/30 Read block 492 len 24576 15/30 Read block 508 len 20480 16/30 Read block 121 len 20480 17/30 Read block 127 len 24576 18/30 Read block 240 len 24576 19/30 Read block 177 len 28672 20/30 Read block 576 len 12288 21/30 Read block 570 len 36864 22/30 Write block 163 len 32768 data: 7 7 9 10 10 9 10 8 23/30 Write block 206 len 8192 data: 13 18 24/30 Write block 337 len 24576 data: 10 11 12 11 10 9 25/30 Read block 51 len 8192 26/30 Read block 237 len 40960 27/30 Read block 275 len 4096 28/30 Read block 100 len 40960 29/30 Read block 151 len 12288 30/30 Read block 357 len 28672 ---------------------------------------------------------------- Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Read 2 Done Done Done false 1002 Acked Read 6 Sent Done Done false 1003 Acked Read 7 Sent Done Done false 1004 Acked Write 4 Sent Done Done false 1005 Acked Read 7 Sent Done Done false 1006 Acked Read 6 Sent Done Done false 1007 Acked Read 3 Sent Done Done false 1008 Acked Read 8 Sent Done Done false 1009 Acked Read 4 Sent Done Done false 1010 Acked Read 10 Sent Done Done false 1011 Acked Write 3 Sent Done Done false 1012 Acked Read 5 Sent Done Done false 1013 Acked Read 6 Sent Done Done false 1014 Acked Read 5 Sent Done Done false 1015 Acked Read 5 Sent Done Done false 1016 Acked Read 6 Sent Done Done false 1017 Acked Read 6 Sent Done Done false 1018 Acked Read 7 Sent Done Done false 1019 Acked Read 3 Sent Done Done false 1020 Acked Read 9 Sent Done Done false 1021 Acked Write 8 Sent Done Done false 1022 Acked Write 2 Sent Done Done false 1023 Acked Write 6 Sent Done Done false 1024 Acked Read 2 Sent Done Done false 1025 Acked Read 10 Sent Done Done false 1026 Acked Read 1 Sent Done Done false 1027 Acked Read 10 Sent Done Done false 1028 Acked Read 3 Sent Done Done false 1029 Acked Read 7 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 0 0 28 Done 2 30 30 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:58:24.947227427Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1696} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:24.947411719Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1696,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 240 240 240 F F F 1 020-039 138 138 138 241 241 241 F F F 2 040-059 136 136 136 238 238 238 T T T 3 060-079 137 137 137 240 240 240 F F F 4 080-099 135 135 135 237 237 237 F F F 5 100-119 136 136 136 238 238 238 F F F 6 120-139 136 136 136 239 239 239 F F F 7 140-159 139 139 139 243 243 243 F F F 8 160-179 137 137 137 240 240 240 F T T <--- 9 180-199 136 136 136 239 239 239 F F F 10 200-219 137 137 137 240 240 240 F T T <--- 11 220-239 138 138 138 241 241 241 F F F 12 240-259 137 137 137 240 240 240 F T T <--- 13 260-279 139 139 139 243 243 243 F F F 14 280-299 138 138 138 242 242 242 F F F 15 300-319 134 134 134 236 236 236 F F F 16 320-339 138 138 138 242 242 242 F T T <--- 17 340-359 137 137 137 240 240 240 F T T <--- 18 360-379 139 139 139 243 243 243 F F F 19 380-399 139 139 139 243 243 243 F F F 20 400-419 133 133 133 232 232 232 F F F 21 420-439 139 139 139 243 243 243 F F F 22 440-459 136 136 136 239 239 239 F F F 23 460-479 139 139 139 243 243 243 F F F 24 480-499 139 139 139 243 243 243 F F F 25 500-519 131 131 131 230 230 230 F T T <--- 26 520-539 135 135 135 237 237 237 F F F 27 540-559 135 135 135 237 237 237 F F F 28 560-579 127 127 127 223 223 223 F F F 29 580-599 138 138 138 241 241 241 F F F Max gen: 139, Max flush: 243 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 70 Verifying data now No change after verify Loop: 70 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 240 240 240 F F F 1 020-039 138 138 138 241 241 241 F F F 2 040-059 140 140 140 244 244 244 F F F 3 060-079 137 137 137 240 240 240 F F F 4 080-099 135 135 135 237 237 237 F F F 5 100-119 136 136 136 238 238 238 F F F 6 120-139 136 136 136 239 239 239 F F F 7 140-159 139 139 139 243 243 243 F F F 8 160-179 140 140 140 244 244 244 F F F 9 180-199 136 136 136 239 239 239 F F F 10 200-219 140 140 140 244 244 244 F F F 11 220-239 138 138 138 241 241 241 F F F 12 240-259 140 140 140 244 244 244 F F F 13 260-279 139 139 139 243 243 243 F F F 14 280-299 138 138 138 242 242 242 F F F 15 300-319 134 134 134 236 236 236 F F F 16 320-339 140 140 140 244 244 244 F F F 17 340-359 140 140 140 244 244 244 F F F 18 360-379 139 139 139 243 243 243 F F F 19 380-399 139 139 139 243 243 243 F F F 20 400-419 133 133 133 232 232 232 F F F 21 420-439 139 139 139 243 243 243 F F F 22 440-459 136 136 136 239 239 239 F F F 23 460-479 139 139 139 243 243 243 F F F 24 480-499 139 139 139 243 243 243 F F F 25 500-519 140 140 140 244 244 244 F F F 26 520-539 135 135 135 237 237 237 F F F 27 540-559 135 135 135 237 237 237 F F F 28 560-579 127 127 127 223 223 223 F F F 29 580-599 138 138 138 241 241 241 F F F Max gen: 140, Max flush: 244 Begin loop 71 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:29.192359948Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1704,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:58:29.199515563Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1704,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:29.199565895Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1704,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:29.20630884Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1704,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:29.20658641Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1704} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:29.206609326Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1704} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:29.206618045Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1704} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:29.206629393Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1704} {"msg":"Crucible 87a445d1-73b1-4dad-917f-1625ebda4b66 has session id: 6e5e10e0-c5f5-4f32-97fe-1dd5b2910b67","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:29.20698724Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1704,"session_id":"6e5e10e0-c5f5-4f32-97fe-1dd5b2910b67"} {"msg":"Upstairs opts: Upstairs UUID: 87a445d1-73b1-4dad-917f-1625ebda4b66, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:29.207025665Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1704,"session_id":"6e5e10e0-c5f5-4f32-97fe-1dd5b2910b67"} {"msg":"Crucible stats registered with UUID: 87a445d1-73b1-4dad-917f-1625ebda4b66","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:29.207037223Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1704,"session_id":"6e5e10e0-c5f5-4f32-97fe-1dd5b2910b67"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:58:29.207046632Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1704,"session_id":"6e5e10e0-c5f5-4f32-97fe-1dd5b2910b67"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:29.207164704Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1704,"":"io task","client":"1","":"downstairs","session_id":"6e5e10e0-c5f5-4f32-97fe-1dd5b2910b67"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:29.207183042Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1704,"":"io task","client":"0","":"downstairs","session_id":"6e5e10e0-c5f5-4f32-97fe-1dd5b2910b67"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:29.20719527Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1704,"":"io task","client":"2","":"downstairs","session_id":"6e5e10e0-c5f5-4f32-97fe-1dd5b2910b67"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:29.207319631Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1704,"task":"crutest"} {"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:29.207337449Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1704} {"msg":"87a445d1-73b1-4dad-917f-1625ebda4b66 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:29.207366544Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1704,"session_id":"6e5e10e0-c5f5-4f32-97fe-1dd5b2910b67"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:29.207375983Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1704,"client":"0","":"downstairs","session_id":"6e5e10e0-c5f5-4f32-97fe-1dd5b2910b67"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:29.207386312Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1704,"client":"1","":"downstairs","session_id":"6e5e10e0-c5f5-4f32-97fe-1dd5b2910b67"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:29.20739622Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1704,"client":"2","":"downstairs","session_id":"6e5e10e0-c5f5-4f32-97fe-1dd5b2910b67"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:29.207406049Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1704,"":"io task","client":"2","":"downstairs","session_id":"6e5e10e0-c5f5-4f32-97fe-1dd5b2910b67"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:29.207416027Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1704,"":"io task","client":"0","":"downstairs","session_id":"6e5e10e0-c5f5-4f32-97fe-1dd5b2910b67"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:29.207426076Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1704,"":"io task","client":"1","":"downstairs","session_id":"6e5e10e0-c5f5-4f32-97fe-1dd5b2910b67"} {"msg":"ds_connection connected from Ok(127.0.0.1:48999)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:29.207483227Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1704,"":"io task","client":"1","":"downstairs","session_id":"6e5e10e0-c5f5-4f32-97fe-1dd5b2910b67"} {"msg":"ds_connection connected from Ok(127.0.0.1:41649)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:29.207508983Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1704,"":"io task","client":"2","":"downstairs","session_id":"6e5e10e0-c5f5-4f32-97fe-1dd5b2910b67"} {"msg":"ds_connection connected from Ok(127.0.0.1:39643)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:29.20759833Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1704,"":"io task","client":"0","":"downstairs","session_id":"6e5e10e0-c5f5-4f32-97fe-1dd5b2910b67"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:29.20821317Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1704,"client":"1","":"downstairs","session_id":"6e5e10e0-c5f5-4f32-97fe-1dd5b2910b67"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:29.208231967Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1704,"client":"2","":"downstairs","session_id":"6e5e10e0-c5f5-4f32-97fe-1dd5b2910b67"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:29.208247294Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1704,"client":"0","":"downstairs","session_id":"6e5e10e0-c5f5-4f32-97fe-1dd5b2910b67"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:29.20855104Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1704,"session_id":"6e5e10e0-c5f5-4f32-97fe-1dd5b2910b67"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:29.208572916Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1704,"session_id":"6e5e10e0-c5f5-4f32-97fe-1dd5b2910b67"} {"msg":"[0]R flush_numbers[0..12]: [240, 241, 244, 240, 237, 238, 239, 243, 244, 239, 244, 241]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:29.208583475Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1704,"":"downstairs","":"downstairs","session_id":"6e5e10e0-c5f5-4f32-97fe-1dd5b2910b67"} {"msg":"[0]R generation[0..12]: [137, 138, 140, 137, 135, 136, 136, 139, 140, 136, 140, 138]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:29.208594213Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1704,"":"downstairs","":"downstairs","session_id":"6e5e10e0-c5f5-4f32-97fe-1dd5b2910b67"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:29.208605172Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1704,"":"downstairs","":"downstairs","session_id":"6e5e10e0-c5f5-4f32-97fe-1dd5b2910b67"} {"msg":"[1]R flush_numbers[0..12]: [240, 241, 244, 240, 237, 238, 239, 243, 244, 239, 244, 241]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:29.20861595Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1704,"":"downstairs","":"downstairs","session_id":"6e5e10e0-c5f5-4f32-97fe-1dd5b2910b67"} {"msg":"[1]R generation[0..12]: [137, 138, 140, 137, 135, 136, 136, 139, 140, 136, 140, 138]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:29.208626599Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1704,"":"downstairs","":"downstairs","session_id":"6e5e10e0-c5f5-4f32-97fe-1dd5b2910b67"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:29.208637527Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1704,"":"downstairs","":"downstairs","session_id":"6e5e10e0-c5f5-4f32-97fe-1dd5b2910b67"} {"msg":"[2]R flush_numbers[0..12]: [240, 241, 244, 240, 237, 238, 239, 243, 244, 239, 244, 241]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:29.208648425Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1704,"":"downstairs","":"downstairs","session_id":"6e5e10e0-c5f5-4f32-97fe-1dd5b2910b67"} {"msg":"[2]R generation[0..12]: [137, 138, 140, 137, 135, 136, 136, 139, 140, 136, 140, 138]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:29.208659224Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1704,"":"downstairs","":"downstairs","session_id":"6e5e10e0-c5f5-4f32-97fe-1dd5b2910b67"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:29.208670122Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1704,"":"downstairs","":"downstairs","session_id":"6e5e10e0-c5f5-4f32-97fe-1dd5b2910b67"} {"msg":"Max found gen is 141","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:29.208679631Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1704,"":"downstairs","":"downstairs","session_id":"6e5e10e0-c5f5-4f32-97fe-1dd5b2910b67"} {"msg":"Generation requested: 142 >= found:141","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:29.208689469Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1704,"":"downstairs","":"downstairs","session_id":"6e5e10e0-c5f5-4f32-97fe-1dd5b2910b67"} {"msg":"Next flush: 245","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:29.208698898Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1704,"":"downstairs","":"downstairs","session_id":"6e5e10e0-c5f5-4f32-97fe-1dd5b2910b67"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:29.208710546Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1704,"":"downstairs","":"downstairs","session_id":"6e5e10e0-c5f5-4f32-97fe-1dd5b2910b67"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:29.208720485Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1704,"session_id":"6e5e10e0-c5f5-4f32-97fe-1dd5b2910b67"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:29.208729153Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1704,"client":"0","":"downstairs","session_id":"6e5e10e0-c5f5-4f32-97fe-1dd5b2910b67"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:29.208738962Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1704,"client":"1","":"downstairs","session_id":"6e5e10e0-c5f5-4f32-97fe-1dd5b2910b67"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:29.208748571Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1704,"client":"2","":"downstairs","session_id":"6e5e10e0-c5f5-4f32-97fe-1dd5b2910b67"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:29.208758299Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1704,"session_id":"6e5e10e0-c5f5-4f32-97fe-1dd5b2910b67"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:29.208767428Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1704,"session_id":"6e5e10e0-c5f5-4f32-97fe-1dd5b2910b67"} {"msg":"87a445d1-73b1-4dad-917f-1625ebda4b66 is now active with session: 6e5e10e0-c5f5-4f32-97fe-1dd5b2910b67","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:29.208777566Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1704,"session_id":"6e5e10e0-c5f5-4f32-97fe-1dd5b2910b67"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:29.208786435Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1704,"session_id":"6e5e10e0-c5f5-4f32-97fe-1dd5b2910b67"} {"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:29.208795544Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1704} {"msg":"Activated sub_volume 87a445d1-73b1-4dad-917f-1625ebda4b66","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:29.208803552Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1704} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:29.208811701Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1704,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:29.20909062Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1704,"task":"crutest"} 01/30 Flush 02/30 Flush 03/30 Write block 275 len 16384 data: 15 12 16 15 04/30 Read block 192 len 28672 05/30 Read block 449 len 4096 06/30 Write block 399 len 28672 data: 10 9 7 5 9 9 10 07/30 Read block 186 len 28672 08/30 Read block 315 len 16384 09/30 Read block 338 len 12288 10/30 Write block 317 len 16384 data: 10 5 7 10 11/30 Write block 470 len 24576 data: 14 13 11 7 10 12 12/30 Write block 239 len 4096 data: 9 13/30 Read block 5 len 20480 14/30 Read block 235 len 40960 15/30 Flush 16/30 Write block 324 len 32768 data: 14 15 16 13 11 11 11 10 17/30 Read block 416 len 12288 18/30 Write block 567 len 28672 data: 5 5 6 8 8 8 9 19/30 Write block 147 len 12288 data: 15 14 13 20/30 Flush 21/30 Write block 372 len 12288 data: 12 10 9 22/30 Read block 560 len 32768 23/30 Write block 428 len 12288 data: 14 13 12 24/30 Read block 191 len 8192 25/30 Read block 193 len 24576 26/30 Write block 356 len 32768 data: 12 13 12 10 12 8 8 13 27/30 Write block 171 len 28672 data: 8 10 10 10 11 13 13 28/30 Write block 185 len 16384 data: 7 6 9 9 29/30 Write block 575 len 24576 data: 10 8 9 11 9 9 30/30 Read block 461 len 4096 ---------------------------------------------------------------- Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 4 Done Done Done false 1003 Acked Read 7 Done Done Done false 1004 Acked Read 1 Done Done Done false 1005 Acked Write 7 Done Done Done false 1006 Acked Read 7 Done Done Done false 1007 Acked Read 4 Sent Done Done false 1008 Acked Read 3 Sent Done Done false 1009 Acked Write 4 Sent Done Done false 1010 Acked Write 6 Sent Done Done false 1011 Acked Write 1 Sent Done Done false 1012 Acked Read 5 Sent Done Done false 1013 Acked Read 10 Sent Done Done false 1014 Acked Flush 0 Sent Done Done false 1015 Acked Write 8 Sent Done Done false 1016 Acked Read 3 Sent Done Done false 1017 Acked Write 7 Sent Done Done false 1018 Acked Write 3 Sent Done Done false 1019 Acked Flush 0 Sent Done Done false 1020 Acked Write 3 Sent Done Done false 1021 Acked Read 8 Sent Done Done false 1022 Acked Write 3 Sent Done Done false 1023 Acked Read 2 Sent Done Done false 1024 Acked Read 6 Sent Done Done false 1025 Acked Write 8 Sent Done Done false 1026 Acked Write 7 Sent Done Done false 1027 Acked Write 4 Sent Done Done false 1028 Acked Write 6 Sent Done Sent false 1029 Acked Read 1 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 23 0 2 25 Done 5 28 26 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1001)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:58:29.217856847Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1704} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:29.218043749Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1704,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 240 240 240 F F F 1 020-039 138 138 138 241 241 241 F F F 2 040-059 140 140 140 244 244 244 F F F 3 060-079 137 137 137 240 240 240 F F F 4 080-099 135 135 135 237 237 237 F F F 5 100-119 136 136 136 238 238 238 F F F 6 120-139 136 136 136 239 239 239 F F F 7 140-159 139 142 142 243 248 248 F F F <--- 8 160-179 140 140 140 244 244 244 F T T <--- 9 180-199 136 136 136 239 239 239 F T T <--- 10 200-219 140 140 140 244 244 244 F F F 11 220-239 138 142 142 241 247 247 F F F <--- 12 240-259 140 140 140 244 244 244 F F F 13 260-279 139 142 142 243 247 247 T F F <--- 14 280-299 138 138 138 242 242 242 F F F 15 300-319 134 142 142 236 247 247 F F F <--- 16 320-339 140 142 142 244 248 248 F F F <--- 17 340-359 140 140 140 244 244 244 F T T <--- 18 360-379 139 139 139 243 243 243 F T T <--- 19 380-399 139 142 142 243 247 247 T F F <--- 20 400-419 133 142 142 232 247 247 T F F <--- 21 420-439 139 139 139 243 243 243 F T T <--- 22 440-459 136 136 136 239 239 239 F F F 23 460-479 139 142 142 243 247 247 F F F <--- 24 480-499 139 139 139 243 243 243 F F F 25 500-519 140 140 140 244 244 244 F F F 26 520-539 135 135 135 237 237 237 F F F 27 540-559 135 135 135 237 237 237 F F F 28 560-579 127 142 142 223 248 248 F T T <--- 29 580-599 138 138 138 241 241 241 F T T <--- Max gen: 142, Max flush: 248 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 71 Verifying data now No change after verify Loop: 71 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 240 240 240 F F F 1 020-039 138 138 138 241 241 241 F F F 2 040-059 140 140 140 244 244 244 F F F 3 060-079 137 137 137 240 240 240 F F F 4 080-099 135 135 135 237 237 237 F F F 5 100-119 136 136 136 238 238 238 F F F 6 120-139 136 136 136 239 239 239 F F F 7 140-159 142 142 142 248 248 248 F F F 8 160-179 143 143 143 249 249 249 F F F 9 180-199 143 143 143 249 249 249 F F F 10 200-219 140 140 140 244 244 244 F F F 11 220-239 142 142 142 247 247 247 F F F 12 240-259 140 140 140 244 244 244 F F F 13 260-279 142 142 142 247 247 247 F F F 14 280-299 138 138 138 242 242 242 F F F 15 300-319 142 142 142 247 247 247 F F F 16 320-339 142 142 142 248 248 248 F F F 17 340-359 143 143 143 249 249 249 F F F 18 360-379 143 143 143 249 249 249 F F F 19 380-399 142 142 142 247 247 247 F F F 20 400-419 142 142 142 247 247 247 F F F 21 420-439 143 143 143 249 249 249 F F F 22 440-459 136 136 136 239 239 239 F F F 23 460-479 142 142 142 247 247 247 F F F 24 480-499 139 139 139 243 243 243 F F F 25 500-519 140 140 140 244 244 244 F F F 26 520-539 135 135 135 237 237 237 F F F 27 540-559 135 135 135 237 237 237 F F F 28 560-579 143 143 143 249 249 249 F F F 29 580-599 143 143 143 249 249 249 F F F Max gen: 143, Max flush: 249 Begin loop 72 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:33.476277488Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1712,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:58:33.483465717Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1712,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:33.483521689Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1712,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:33.490367109Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1712,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:33.490637119Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1712} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:33.490662456Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1712} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:33.490670774Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1712} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:33.490682203Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1712} {"msg":"Crucible f43cdef3-b25b-4110-8790-209958936ba2 has session id: 60459917-003a-4b70-8f31-7eeb4e033b0a","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:33.491015994Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1712,"session_id":"60459917-003a-4b70-8f31-7eeb4e033b0a"} {"msg":"Upstairs opts: Upstairs UUID: f43cdef3-b25b-4110-8790-209958936ba2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:33.491046459Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1712,"session_id":"60459917-003a-4b70-8f31-7eeb4e033b0a"} {"msg":"Crucible stats registered with UUID: f43cdef3-b25b-4110-8790-209958936ba2","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:33.491057587Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1712,"session_id":"60459917-003a-4b70-8f31-7eeb4e033b0a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:58:33.491066836Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1712,"session_id":"60459917-003a-4b70-8f31-7eeb4e033b0a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:33.491185189Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1712,"":"io task","client":"2","":"downstairs","session_id":"60459917-003a-4b70-8f31-7eeb4e033b0a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:33.491203426Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1712,"":"io task","client":"0","":"downstairs","session_id":"60459917-003a-4b70-8f31-7eeb4e033b0a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:33.491213884Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1712,"":"io task","client":"1","":"downstairs","session_id":"60459917-003a-4b70-8f31-7eeb4e033b0a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:33.491347355Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1712,"task":"crutest"} {"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:33.491365452Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1712} {"msg":"f43cdef3-b25b-4110-8790-209958936ba2 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:33.491383059Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1712,"session_id":"60459917-003a-4b70-8f31-7eeb4e033b0a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:33.491393688Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1712,"client":"0","":"downstairs","session_id":"60459917-003a-4b70-8f31-7eeb4e033b0a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:33.491403626Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1712,"client":"1","":"downstairs","session_id":"60459917-003a-4b70-8f31-7eeb4e033b0a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:33.491413305Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1712,"client":"2","":"downstairs","session_id":"60459917-003a-4b70-8f31-7eeb4e033b0a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:33.491422854Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1712,"":"io task","client":"2","":"downstairs","session_id":"60459917-003a-4b70-8f31-7eeb4e033b0a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:33.491432962Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1712,"":"io task","client":"0","":"downstairs","session_id":"60459917-003a-4b70-8f31-7eeb4e033b0a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:33.491442811Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1712,"":"io task","client":"1","":"downstairs","session_id":"60459917-003a-4b70-8f31-7eeb4e033b0a"} {"msg":"ds_connection connected from Ok(127.0.0.1:40035)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:33.491496143Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1712,"":"io task","client":"1","":"downstairs","session_id":"60459917-003a-4b70-8f31-7eeb4e033b0a"} {"msg":"ds_connection connected from Ok(127.0.0.1:52013)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:33.491546395Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1712,"":"io task","client":"0","":"downstairs","session_id":"60459917-003a-4b70-8f31-7eeb4e033b0a"} {"msg":"ds_connection connected from Ok(127.0.0.1:45109)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:33.491557854Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1712,"":"io task","client":"2","":"downstairs","session_id":"60459917-003a-4b70-8f31-7eeb4e033b0a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:33.492239623Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1712,"client":"1","":"downstairs","session_id":"60459917-003a-4b70-8f31-7eeb4e033b0a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:33.492282487Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1712,"client":"2","":"downstairs","session_id":"60459917-003a-4b70-8f31-7eeb4e033b0a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:33.492298644Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1712,"client":"0","":"downstairs","session_id":"60459917-003a-4b70-8f31-7eeb4e033b0a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:33.492586252Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1712,"session_id":"60459917-003a-4b70-8f31-7eeb4e033b0a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:33.49259631Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1712,"session_id":"60459917-003a-4b70-8f31-7eeb4e033b0a"} {"msg":"[0]R flush_numbers[0..12]: [240, 241, 244, 240, 237, 238, 239, 248, 249, 249, 244, 247]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:33.492611378Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1712,"":"downstairs","":"downstairs","session_id":"60459917-003a-4b70-8f31-7eeb4e033b0a"} {"msg":"[0]R generation[0..12]: [137, 138, 140, 137, 135, 136, 136, 142, 143, 143, 140, 142]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:33.492624246Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1712,"":"downstairs","":"downstairs","session_id":"60459917-003a-4b70-8f31-7eeb4e033b0a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:33.492635395Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1712,"":"downstairs","":"downstairs","session_id":"60459917-003a-4b70-8f31-7eeb4e033b0a"} {"msg":"[1]R flush_numbers[0..12]: [240, 241, 244, 240, 237, 238, 239, 248, 249, 249, 244, 247]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:33.492646113Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1712,"":"downstairs","":"downstairs","session_id":"60459917-003a-4b70-8f31-7eeb4e033b0a"} {"msg":"[1]R generation[0..12]: [137, 138, 140, 137, 135, 136, 136, 142, 143, 143, 140, 142]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:33.492656831Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1712,"":"downstairs","":"downstairs","session_id":"60459917-003a-4b70-8f31-7eeb4e033b0a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:33.49266784Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1712,"":"downstairs","":"downstairs","session_id":"60459917-003a-4b70-8f31-7eeb4e033b0a"} {"msg":"[2]R flush_numbers[0..12]: [240, 241, 244, 240, 237, 238, 239, 248, 249, 249, 244, 247]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:33.492678598Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1712,"":"downstairs","":"downstairs","session_id":"60459917-003a-4b70-8f31-7eeb4e033b0a"} {"msg":"[2]R generation[0..12]: [137, 138, 140, 137, 135, 136, 136, 142, 143, 143, 140, 142]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:33.492689367Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1712,"":"downstairs","":"downstairs","session_id":"60459917-003a-4b70-8f31-7eeb4e033b0a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:33.492700335Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1712,"":"downstairs","":"downstairs","session_id":"60459917-003a-4b70-8f31-7eeb4e033b0a"} {"msg":"Max found gen is 144","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:33.492709844Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1712,"":"downstairs","":"downstairs","session_id":"60459917-003a-4b70-8f31-7eeb4e033b0a"} {"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:33.492719772Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1712,"":"downstairs","":"downstairs","session_id":"60459917-003a-4b70-8f31-7eeb4e033b0a"} {"msg":"Next flush: 250","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:33.492729611Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1712,"":"downstairs","":"downstairs","session_id":"60459917-003a-4b70-8f31-7eeb4e033b0a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:33.492739139Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1712,"":"downstairs","":"downstairs","session_id":"60459917-003a-4b70-8f31-7eeb4e033b0a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:33.492748928Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1712,"session_id":"60459917-003a-4b70-8f31-7eeb4e033b0a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:33.492757807Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1712,"client":"0","":"downstairs","session_id":"60459917-003a-4b70-8f31-7eeb4e033b0a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:33.492767675Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1712,"client":"1","":"downstairs","session_id":"60459917-003a-4b70-8f31-7eeb4e033b0a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:33.492778484Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1712,"client":"2","":"downstairs","session_id":"60459917-003a-4b70-8f31-7eeb4e033b0a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:33.492788492Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1712,"session_id":"60459917-003a-4b70-8f31-7eeb4e033b0a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:33.492797541Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1712,"session_id":"60459917-003a-4b70-8f31-7eeb4e033b0a"} {"msg":"f43cdef3-b25b-4110-8790-209958936ba2 is now active with session: 60459917-003a-4b70-8f31-7eeb4e033b0a","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:33.492807529Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1712,"session_id":"60459917-003a-4b70-8f31-7eeb4e033b0a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:33.492816288Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1712,"session_id":"60459917-003a-4b70-8f31-7eeb4e033b0a"} {"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:33.492825577Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1712} {"msg":"Activated sub_volume f43cdef3-b25b-4110-8790-209958936ba2","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:33.492833545Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1712} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:33.492841694Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1712,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:33.493126662Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1712,"task":"crutest"} 01/30 Write block 559 len 8192 data: 8 7 02/30 Write block 525 len 16384 data: 5 5 7 8 03/30 Read block 566 len 28672 04/30 Read block 172 len 12288 05/30 Read block 56 len 32768 06/30 Write block 288 len 4096 data: 12 07/30 Read block 356 len 12288 08/30 Write block 308 len 16384 data: 9 12 14 17 09/30 Write block 250 len 28672 data: 8 9 10 12 15 13 15 10/30 Read block 515 len 28672 11/30 Read block 478 len 36864 12/30 Write block 52 len 28672 data: 11 9 12 11 13 9 8 13/30 Read block 270 len 40960 14/30 Read block 198 len 4096 15/30 Read block 455 len 12288 16/30 Read block 472 len 28672 17/30 Read block 332 len 28672 18/30 Write block 513 len 28672 data: 15 15 14 13 12 15 13 19/30 Read block 196 len 40960 20/30 Flush 21/30 Write block 496 len 40960 data: 9 8 7 10 11 10 12 12 10 10 22/30 Read block 161 len 32768 23/30 Read block 406 len 24576 24/30 Write block 593 len 24576 data: 13 13 12 11 8 5 25/30 Write block 526 len 24576 data: 6 8 9 7 8 9 26/30 Read block 362 len 24576 27/30 Write block 576 len 16384 data: 9 10 12 10 28/30 Read block 373 len 40960 29/30 Write block 307 len 40960 data: 8 10 13 15 18 14 14 11 14 15 30/30 Read block 514 len 8192 ---------------------------------------------------------------- Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Write 4 Sent Done Done false 1002 Acked Read 7 Sent Done Done false 1003 Acked Read 3 Sent Done Done false 1004 Acked Read 8 Sent Done Done false 1005 Acked Write 1 Sent Done Done false 1006 Acked Read 3 Sent Done Done false 1007 Acked Write 4 Sent Done Done false 1008 Acked Write 7 Sent Done Done false 1009 Acked Read 7 Sent Done Done false 1010 Acked Read 9 Sent Done Done false 1011 Acked Write 7 Sent Done Done false 1012 Acked Read 10 Sent Done Done false 1013 Acked Read 1 Sent Done Done false 1014 Acked Read 3 Sent Done Done false 1015 Acked Read 7 Sent Done Done false 1016 Acked Read 7 Sent Done Done false 1017 Acked Write 7 Sent Done Done false 1018 Acked Read 10 Sent Done Done false 1019 Acked Flush 0 Sent Done Done false 1020 Acked Write 10 Sent Done Done false 1021 Acked Read 8 Sent Done Done false 1022 Acked Read 6 Sent Done Done false 1023 Acked Write 6 Sent Done Done false 1024 Acked Write 6 Sent Done Done false 1025 Acked Read 6 Sent Done Done false 1026 Acked Write 4 Sent Done Done false 1027 Acked Read 10 Sent Done Done false 1028 Acked Write 10 Sent Done Done false 1029 Acked Read 2 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 0 0 29 Done 1 30 30 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:58:33.502962641Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1712} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:33.503144974Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1712,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 240 240 240 F F F 1 020-039 138 138 138 241 241 241 F F F 2 040-059 140 144 144 244 250 250 F F F <--- 3 060-079 137 137 137 240 240 240 F F F 4 080-099 135 135 135 237 237 237 F F F 5 100-119 136 136 136 238 238 238 F F F 6 120-139 136 136 136 239 239 239 F F F 7 140-159 142 142 142 248 248 248 F F F 8 160-179 143 143 143 249 249 249 F F F 9 180-199 143 143 143 249 249 249 F F F 10 200-219 140 140 140 244 244 244 F F F 11 220-239 142 142 142 247 247 247 F F F 12 240-259 140 144 144 244 250 250 F F F <--- 13 260-279 142 142 142 247 247 247 F F F 14 280-299 138 144 144 242 250 250 F F F <--- 15 300-319 142 144 144 247 250 250 F T T <--- 16 320-339 142 142 142 248 248 248 F F F 17 340-359 143 143 143 249 249 249 F F F 18 360-379 143 143 143 249 249 249 F F F 19 380-399 142 142 142 247 247 247 F F F 20 400-419 142 142 142 247 247 247 F F F 21 420-439 143 143 143 249 249 249 F F F 22 440-459 136 136 136 239 239 239 F F F 23 460-479 142 142 142 247 247 247 F F F 24 480-499 139 139 139 243 243 243 F T T <--- 25 500-519 140 144 144 244 250 250 F T T <--- 26 520-539 135 144 144 237 250 250 F T T <--- 27 540-559 135 144 144 237 250 250 T F F <--- 28 560-579 143 144 144 249 250 250 T T T <--- 29 580-599 143 143 143 249 249 249 F T T <--- Max gen: 144, Max flush: 250 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 72 Verifying data now No change after verify Loop: 72 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 240 240 240 F F F 1 020-039 138 138 138 241 241 241 F F F 2 040-059 144 144 144 250 250 250 F F F 3 060-079 137 137 137 240 240 240 F F F 4 080-099 135 135 135 237 237 237 F F F 5 100-119 136 136 136 238 238 238 F F F 6 120-139 136 136 136 239 239 239 F F F 7 140-159 142 142 142 248 248 248 F F F 8 160-179 143 143 143 249 249 249 F F F 9 180-199 143 143 143 249 249 249 F F F 10 200-219 140 140 140 244 244 244 F F F 11 220-239 142 142 142 247 247 247 F F F 12 240-259 144 144 144 250 250 250 F F F 13 260-279 142 142 142 247 247 247 F F F 14 280-299 144 144 144 250 250 250 F F F 15 300-319 145 145 145 251 251 251 F F F 16 320-339 142 142 142 248 248 248 F F F 17 340-359 143 143 143 249 249 249 F F F 18 360-379 143 143 143 249 249 249 F F F 19 380-399 142 142 142 247 247 247 F F F 20 400-419 142 142 142 247 247 247 F F F 21 420-439 143 143 143 249 249 249 F F F 22 440-459 136 136 136 239 239 239 F F F 23 460-479 142 142 142 247 247 247 F F F 24 480-499 145 145 145 251 251 251 F F F 25 500-519 145 145 145 251 251 251 F F F 26 520-539 145 145 145 251 251 251 F F F 27 540-559 144 144 144 250 250 250 F F F 28 560-579 145 145 145 251 251 251 F F F 29 580-599 145 145 145 251 251 251 F F F Max gen: 145, Max flush: 251 Begin loop 73 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:37.752319729Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1720,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:58:37.759414773Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1720,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:37.759465745Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1720,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:37.766172986Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1720,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:37.766434277Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1720} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:37.766463023Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1720} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:37.766471842Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1720} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:37.766486099Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1720} {"msg":"Crucible f0be9416-ae47-4bd1-91cb-38a55a40bb66 has session id: de7844de-4758-4d92-96a8-12620204ba66","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:37.766827819Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1720,"session_id":"de7844de-4758-4d92-96a8-12620204ba66"} {"msg":"Upstairs opts: Upstairs UUID: f0be9416-ae47-4bd1-91cb-38a55a40bb66, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:37.76688822Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1720,"session_id":"de7844de-4758-4d92-96a8-12620204ba66"} {"msg":"Crucible stats registered with UUID: f0be9416-ae47-4bd1-91cb-38a55a40bb66","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:37.766898999Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1720,"session_id":"de7844de-4758-4d92-96a8-12620204ba66"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:58:37.766908007Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1720,"session_id":"de7844de-4758-4d92-96a8-12620204ba66"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:37.767055295Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1720,"":"io task","client":"1","":"downstairs","session_id":"de7844de-4758-4d92-96a8-12620204ba66"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:37.767086401Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1720,"":"io task","client":"0","":"downstairs","session_id":"de7844de-4758-4d92-96a8-12620204ba66"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:37.767097539Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1720,"":"io task","client":"2","":"downstairs","session_id":"de7844de-4758-4d92-96a8-12620204ba66"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:37.767221941Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1720,"task":"crutest"} {"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:37.767236979Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1720} {"msg":"f0be9416-ae47-4bd1-91cb-38a55a40bb66 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:37.767260495Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1720,"session_id":"de7844de-4758-4d92-96a8-12620204ba66"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:37.767269964Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1720,"client":"0","":"downstairs","session_id":"de7844de-4758-4d92-96a8-12620204ba66"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:37.767279682Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1720,"client":"1","":"downstairs","session_id":"de7844de-4758-4d92-96a8-12620204ba66"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:37.767289321Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1720,"client":"2","":"downstairs","session_id":"de7844de-4758-4d92-96a8-12620204ba66"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:37.76729873Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1720,"":"io task","client":"2","":"downstairs","session_id":"de7844de-4758-4d92-96a8-12620204ba66"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:37.767311558Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1720,"":"io task","client":"0","":"downstairs","session_id":"de7844de-4758-4d92-96a8-12620204ba66"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:37.767321726Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1720,"":"io task","client":"1","":"downstairs","session_id":"de7844de-4758-4d92-96a8-12620204ba66"} {"msg":"ds_connection connected from Ok(127.0.0.1:56684)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:37.767375888Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1720,"":"io task","client":"2","":"downstairs","session_id":"de7844de-4758-4d92-96a8-12620204ba66"} {"msg":"ds_connection connected from Ok(127.0.0.1:37539)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:37.767386997Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1720,"":"io task","client":"1","":"downstairs","session_id":"de7844de-4758-4d92-96a8-12620204ba66"} {"msg":"ds_connection connected from Ok(127.0.0.1:56659)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:37.767397315Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1720,"":"io task","client":"0","":"downstairs","session_id":"de7844de-4758-4d92-96a8-12620204ba66"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:37.768059337Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1720,"client":"1","":"downstairs","session_id":"de7844de-4758-4d92-96a8-12620204ba66"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:37.76817972Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1720,"client":"2","":"downstairs","session_id":"de7844de-4758-4d92-96a8-12620204ba66"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:37.768301032Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1720,"client":"0","":"downstairs","session_id":"de7844de-4758-4d92-96a8-12620204ba66"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:37.768457859Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1720,"session_id":"de7844de-4758-4d92-96a8-12620204ba66"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:37.768526348Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1720,"session_id":"de7844de-4758-4d92-96a8-12620204ba66"} {"msg":"[0]R flush_numbers[0..12]: [240, 241, 250, 240, 237, 238, 239, 248, 249, 249, 244, 247]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:37.768559644Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1720,"":"downstairs","":"downstairs","session_id":"de7844de-4758-4d92-96a8-12620204ba66"} {"msg":"[0]R generation[0..12]: [137, 138, 144, 137, 135, 136, 136, 142, 143, 143, 140, 142]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:37.768571822Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1720,"":"downstairs","":"downstairs","session_id":"de7844de-4758-4d92-96a8-12620204ba66"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:37.768583Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1720,"":"downstairs","":"downstairs","session_id":"de7844de-4758-4d92-96a8-12620204ba66"} {"msg":"[1]R flush_numbers[0..12]: [240, 241, 250, 240, 237, 238, 239, 248, 249, 249, 244, 247]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:37.768594058Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1720,"":"downstairs","":"downstairs","session_id":"de7844de-4758-4d92-96a8-12620204ba66"} {"msg":"[1]R generation[0..12]: [137, 138, 144, 137, 135, 136, 136, 142, 143, 143, 140, 142]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:37.768606757Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1720,"":"downstairs","":"downstairs","session_id":"de7844de-4758-4d92-96a8-12620204ba66"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:37.768617965Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1720,"":"downstairs","":"downstairs","session_id":"de7844de-4758-4d92-96a8-12620204ba66"} {"msg":"[2]R flush_numbers[0..12]: [240, 241, 250, 240, 237, 238, 239, 248, 249, 249, 244, 247]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:37.768628573Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1720,"":"downstairs","":"downstairs","session_id":"de7844de-4758-4d92-96a8-12620204ba66"} {"msg":"[2]R generation[0..12]: [137, 138, 144, 137, 135, 136, 136, 142, 143, 143, 140, 142]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:37.768639112Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1720,"":"downstairs","":"downstairs","session_id":"de7844de-4758-4d92-96a8-12620204ba66"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:37.768655169Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1720,"":"downstairs","":"downstairs","session_id":"de7844de-4758-4d92-96a8-12620204ba66"} {"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:37.768664748Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1720,"":"downstairs","":"downstairs","session_id":"de7844de-4758-4d92-96a8-12620204ba66"} {"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:37.768674307Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1720,"":"downstairs","":"downstairs","session_id":"de7844de-4758-4d92-96a8-12620204ba66"} {"msg":"Next flush: 252","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:37.768683835Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1720,"":"downstairs","":"downstairs","session_id":"de7844de-4758-4d92-96a8-12620204ba66"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:37.768693284Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1720,"":"downstairs","":"downstairs","session_id":"de7844de-4758-4d92-96a8-12620204ba66"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:37.768703052Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1720,"session_id":"de7844de-4758-4d92-96a8-12620204ba66"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:37.768711951Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1720,"client":"0","":"downstairs","session_id":"de7844de-4758-4d92-96a8-12620204ba66"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:37.76872167Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1720,"client":"1","":"downstairs","session_id":"de7844de-4758-4d92-96a8-12620204ba66"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:37.768731668Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1720,"client":"2","":"downstairs","session_id":"de7844de-4758-4d92-96a8-12620204ba66"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:37.768741697Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1720,"session_id":"de7844de-4758-4d92-96a8-12620204ba66"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:37.768751045Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1720,"session_id":"de7844de-4758-4d92-96a8-12620204ba66"} {"msg":"f0be9416-ae47-4bd1-91cb-38a55a40bb66 is now active with session: de7844de-4758-4d92-96a8-12620204ba66","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:37.768762054Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1720,"session_id":"de7844de-4758-4d92-96a8-12620204ba66"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:37.768770752Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1720,"session_id":"de7844de-4758-4d92-96a8-12620204ba66"} {"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:37.768779621Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1720} {"msg":"Activated sub_volume f0be9416-ae47-4bd1-91cb-38a55a40bb66","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:37.76878769Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1720} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:37.768795829Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1720,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:37.76912658Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1720,"task":"crutest"} 01/30 Write block 504 len 4096 data: 11 02/30 Flush 03/30 Write block 524 len 16384 data: 7 6 7 9 04/30 Read block 422 len 12288 05/30 Read block 288 len 28672 06/30 Read block 11 len 16384 07/30 Write block 379 len 28672 data: 9 9 11 15 15 12 10 08/30 Flush 09/30 Write block 128 len 36864 data: 6 8 7 7 6 9 13 12 10 10/30 Flush 11/30 Write block 192 len 20480 data: 17 17 16 16 12 12/30 Write block 251 len 36864 data: 10 11 13 16 14 16 17 19 16 13/30 Read block 247 len 12288 14/30 Write block 137 len 4096 data: 8 15/30 Read block 583 len 24576 16/30 Read block 361 len 16384 17/30 Read block 167 len 4096 18/30 Read block 479 len 32768 19/30 Write block 172 len 24576 data: 11 11 11 12 14 14 20/30 Read block 416 len 20480 21/30 Read block 431 len 24576 22/30 Write block 433 len 4096 data: 12 23/30 Flush 24/30 Write block 121 len 4096 data: 16 25/30 Read block 134 len 28672 26/30 Flush 27/30 Write block 1 len 8192 data: 2 2 28/30 Read block 352 len 8192 29/30 Write block 131 len 16384 data: 8 7 10 14 30/30 Write block 68 len 40960 data: 8 7 7 8 7 7 8 8 8 8 ---------------------------------------------------------------- Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 4 Sent Done Done false 1003 Acked Read 3 Sent Done Done false 1004 Acked Read 7 Sent Done Done false 1005 Acked Read 4 Sent Done Done false 1006 Acked Write 7 Sent Done Done false 1007 Acked Flush 0 Sent Done Done false 1008 Acked Write 9 Sent Done Done false 1009 Acked Flush 0 Sent Done Done false 1010 Acked Write 5 Sent Done Done false 1011 Acked Write 9 Sent Done Done false 1012 Acked Read 3 Sent Done Done false 1013 Acked Write 1 Sent Done Done false 1014 Acked Read 6 Sent Done Done false 1015 Acked Read 4 Sent Done Done false 1016 Acked Read 1 Sent Done Done false 1017 Acked Read 8 Sent Done Done false 1018 Acked Write 6 Sent Done Done false 1019 Acked Read 5 Sent Done Done false 1020 Acked Read 6 Sent Done Done false 1021 Acked Write 1 Sent Done Done false 1022 Acked Flush 0 Sent Done Done false 1023 Acked Write 1 Sent Done Done false 1024 Acked Read 7 Sent Done Done false 1025 Acked Flush 0 Sent Done Done false 1026 Acked Write 2 Sent Done Done false 1027 Acked Read 2 Sent Done Done false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 2 2 32 Done 0 26 26 52 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1001)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:58:37.777674839Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1720} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:37.777865041Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1720,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 240 240 240 F T T <--- 1 020-039 138 138 138 241 241 241 F F F 2 040-059 144 144 144 250 250 250 F F F 3 060-079 137 137 137 240 240 240 F T T <--- 4 080-099 135 135 135 237 237 237 F F F 5 100-119 136 136 136 238 238 238 F F F 6 120-139 136 146 146 239 256 256 F T T <--- 7 140-159 142 142 142 248 248 248 F F F 8 160-179 143 146 146 249 255 255 F F F <--- 9 180-199 143 146 146 249 255 255 F F F <--- 10 200-219 140 140 140 244 244 244 F F F 11 220-239 142 142 142 247 247 247 F F F 12 240-259 144 146 146 250 255 255 F F F <--- 13 260-279 142 142 142 247 247 247 F F F 14 280-299 144 144 144 250 250 250 F F F 15 300-319 145 145 145 251 251 251 F F F 16 320-339 142 142 142 248 248 248 F F F 17 340-359 143 143 143 249 249 249 F F F 18 360-379 143 146 146 249 253 253 F F F <--- 19 380-399 142 146 146 247 253 253 F F F <--- 20 400-419 142 142 142 247 247 247 F F F 21 420-439 143 146 146 249 255 255 F F F <--- 22 440-459 136 136 136 239 239 239 F F F 23 460-479 142 142 142 247 247 247 F F F 24 480-499 145 145 145 251 251 251 F F F 25 500-519 146 146 146 252 252 252 F F F 26 520-539 145 146 146 251 253 253 F F F <--- 27 540-559 144 144 144 250 250 250 F F F 28 560-579 145 145 145 251 251 251 F F F 29 580-599 145 145 145 251 251 251 F F F Max gen: 146, Max flush: 256 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 73 Verifying data now No change after verify Loop: 73 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 147 147 147 257 257 257 F F F 1 020-039 138 138 138 241 241 241 F F F 2 040-059 144 144 144 250 250 250 F F F 3 060-079 147 147 147 257 257 257 F F F 4 080-099 135 135 135 237 237 237 F F F 5 100-119 136 136 136 238 238 238 F F F 6 120-139 147 147 147 257 257 257 F F F 7 140-159 142 142 142 248 248 248 F F F 8 160-179 146 146 146 255 255 255 F F F 9 180-199 146 146 146 255 255 255 F F F 10 200-219 140 140 140 244 244 244 F F F 11 220-239 142 142 142 247 247 247 F F F 12 240-259 146 146 146 255 255 255 F F F 13 260-279 142 142 142 247 247 247 F F F 14 280-299 144 144 144 250 250 250 F F F 15 300-319 145 145 145 251 251 251 F F F 16 320-339 142 142 142 248 248 248 F F F 17 340-359 143 143 143 249 249 249 F F F 18 360-379 146 146 146 253 253 253 F F F 19 380-399 146 146 146 253 253 253 F F F 20 400-419 142 142 142 247 247 247 F F F 21 420-439 146 146 146 255 255 255 F F F 22 440-459 136 136 136 239 239 239 F F F 23 460-479 142 142 142 247 247 247 F F F 24 480-499 145 145 145 251 251 251 F F F 25 500-519 146 146 146 252 252 252 F F F 26 520-539 146 146 146 253 253 253 F F F 27 540-559 144 144 144 250 250 250 F F F 28 560-579 145 145 145 251 251 251 F F F 29 580-599 145 145 145 251 251 251 F F F Max gen: 147, Max flush: 257 Begin loop 74 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:42.027305426Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1728,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:58:42.034470149Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1728,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:42.03453481Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1728,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:42.041571151Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1728,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:42.041856339Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1728} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:42.041884485Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1728} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:42.041892924Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1728} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:42.041904402Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1728} {"msg":"Crucible aff22b3a-5abb-4e45-8f9c-c571526928a2 has session id: ac664eb1-d743-45d3-952b-f7bf7209b83a","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:42.042267619Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1728,"session_id":"ac664eb1-d743-45d3-952b-f7bf7209b83a"} {"msg":"Upstairs opts: Upstairs UUID: aff22b3a-5abb-4e45-8f9c-c571526928a2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:42.042296464Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1728,"session_id":"ac664eb1-d743-45d3-952b-f7bf7209b83a"} {"msg":"Crucible stats registered with UUID: aff22b3a-5abb-4e45-8f9c-c571526928a2","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:42.042310072Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1728,"session_id":"ac664eb1-d743-45d3-952b-f7bf7209b83a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:58:42.042319371Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1728,"session_id":"ac664eb1-d743-45d3-952b-f7bf7209b83a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:42.042477088Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1728,"":"io task","client":"2","":"downstairs","session_id":"ac664eb1-d743-45d3-952b-f7bf7209b83a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:42.042508543Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1728,"":"io task","client":"0","":"downstairs","session_id":"ac664eb1-d743-45d3-952b-f7bf7209b83a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:42.042519601Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1728,"":"io task","client":"1","":"downstairs","session_id":"ac664eb1-d743-45d3-952b-f7bf7209b83a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:42.042637864Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1728,"task":"crutest"} {"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:42.042653612Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1728} {"msg":"aff22b3a-5abb-4e45-8f9c-c571526928a2 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:42.042691416Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1728,"session_id":"ac664eb1-d743-45d3-952b-f7bf7209b83a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:42.042701115Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1728,"client":"0","":"downstairs","session_id":"ac664eb1-d743-45d3-952b-f7bf7209b83a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:42.042710843Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1728,"client":"1","":"downstairs","session_id":"ac664eb1-d743-45d3-952b-f7bf7209b83a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:42.042720492Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1728,"client":"2","":"downstairs","session_id":"ac664eb1-d743-45d3-952b-f7bf7209b83a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:42.04273015Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1728,"":"io task","client":"2","":"downstairs","session_id":"ac664eb1-d743-45d3-952b-f7bf7209b83a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:42.042740559Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1728,"":"io task","client":"0","":"downstairs","session_id":"ac664eb1-d743-45d3-952b-f7bf7209b83a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:42.042750647Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1728,"":"io task","client":"1","":"downstairs","session_id":"ac664eb1-d743-45d3-952b-f7bf7209b83a"} {"msg":"ds_connection connected from Ok(127.0.0.1:46277)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:42.042795501Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1728,"":"io task","client":"2","":"downstairs","session_id":"ac664eb1-d743-45d3-952b-f7bf7209b83a"} {"msg":"ds_connection connected from Ok(127.0.0.1:45299)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:42.042821107Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1728,"":"io task","client":"0","":"downstairs","session_id":"ac664eb1-d743-45d3-952b-f7bf7209b83a"} {"msg":"ds_connection connected from Ok(127.0.0.1:55082)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:42.042834255Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1728,"":"io task","client":"1","":"downstairs","session_id":"ac664eb1-d743-45d3-952b-f7bf7209b83a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:42.043588354Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1728,"client":"2","":"downstairs","session_id":"ac664eb1-d743-45d3-952b-f7bf7209b83a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:42.04361255Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1728,"client":"1","":"downstairs","session_id":"ac664eb1-d743-45d3-952b-f7bf7209b83a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:42.043657794Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1728,"client":"0","":"downstairs","session_id":"ac664eb1-d743-45d3-952b-f7bf7209b83a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:42.043933653Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1728,"session_id":"ac664eb1-d743-45d3-952b-f7bf7209b83a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:42.04395483Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1728,"session_id":"ac664eb1-d743-45d3-952b-f7bf7209b83a"} {"msg":"[0]R flush_numbers[0..12]: [257, 241, 250, 257, 237, 238, 257, 248, 255, 255, 244, 247]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:42.044047986Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1728,"":"downstairs","":"downstairs","session_id":"ac664eb1-d743-45d3-952b-f7bf7209b83a"} {"msg":"[0]R generation[0..12]: [147, 138, 144, 147, 135, 136, 147, 142, 146, 146, 140, 142]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:42.044059824Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1728,"":"downstairs","":"downstairs","session_id":"ac664eb1-d743-45d3-952b-f7bf7209b83a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:42.044071153Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1728,"":"downstairs","":"downstairs","session_id":"ac664eb1-d743-45d3-952b-f7bf7209b83a"} {"msg":"[1]R flush_numbers[0..12]: [257, 241, 250, 257, 237, 238, 257, 248, 255, 255, 244, 247]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:42.044082021Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1728,"":"downstairs","":"downstairs","session_id":"ac664eb1-d743-45d3-952b-f7bf7209b83a"} {"msg":"[1]R generation[0..12]: [147, 138, 144, 147, 135, 136, 147, 142, 146, 146, 140, 142]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:42.044092459Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1728,"":"downstairs","":"downstairs","session_id":"ac664eb1-d743-45d3-952b-f7bf7209b83a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:42.044103598Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1728,"":"downstairs","":"downstairs","session_id":"ac664eb1-d743-45d3-952b-f7bf7209b83a"} {"msg":"[2]R flush_numbers[0..12]: [257, 241, 250, 257, 237, 238, 257, 248, 255, 255, 244, 247]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:42.044114336Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1728,"":"downstairs","":"downstairs","session_id":"ac664eb1-d743-45d3-952b-f7bf7209b83a"} {"msg":"[2]R generation[0..12]: [147, 138, 144, 147, 135, 136, 147, 142, 146, 146, 140, 142]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:42.044125215Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1728,"":"downstairs","":"downstairs","session_id":"ac664eb1-d743-45d3-952b-f7bf7209b83a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:42.044138143Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1728,"":"downstairs","":"downstairs","session_id":"ac664eb1-d743-45d3-952b-f7bf7209b83a"} {"msg":"Max found gen is 148","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:42.044147781Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1728,"":"downstairs","":"downstairs","session_id":"ac664eb1-d743-45d3-952b-f7bf7209b83a"} {"msg":"Generation requested: 148 >= found:148","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:42.04415757Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1728,"":"downstairs","":"downstairs","session_id":"ac664eb1-d743-45d3-952b-f7bf7209b83a"} {"msg":"Next flush: 258","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:42.044167098Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1728,"":"downstairs","":"downstairs","session_id":"ac664eb1-d743-45d3-952b-f7bf7209b83a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:42.044176487Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1728,"":"downstairs","":"downstairs","session_id":"ac664eb1-d743-45d3-952b-f7bf7209b83a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:42.044186356Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1728,"session_id":"ac664eb1-d743-45d3-952b-f7bf7209b83a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:42.044195044Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1728,"client":"0","":"downstairs","session_id":"ac664eb1-d743-45d3-952b-f7bf7209b83a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:42.044205083Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1728,"client":"1","":"downstairs","session_id":"ac664eb1-d743-45d3-952b-f7bf7209b83a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:42.044214861Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1728,"client":"2","":"downstairs","session_id":"ac664eb1-d743-45d3-952b-f7bf7209b83a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:42.04422484Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1728,"session_id":"ac664eb1-d743-45d3-952b-f7bf7209b83a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:42.044234019Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1728,"session_id":"ac664eb1-d743-45d3-952b-f7bf7209b83a"} {"msg":"aff22b3a-5abb-4e45-8f9c-c571526928a2 is now active with session: ac664eb1-d743-45d3-952b-f7bf7209b83a","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:42.044244267Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1728,"session_id":"ac664eb1-d743-45d3-952b-f7bf7209b83a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:42.044253126Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1728,"session_id":"ac664eb1-d743-45d3-952b-f7bf7209b83a"} {"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:42.044262354Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1728} {"msg":"Activated sub_volume aff22b3a-5abb-4e45-8f9c-c571526928a2","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:42.044270263Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1728} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:42.044278632Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1728,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:42.044618002Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1728,"task":"crutest"} 01/30 Write block 100 len 40960 data: 17 18 12 14 12 12 9 10 10 7 02/30 Read block 150 len 8192 03/30 Read block 264 len 24576 04/30 Write block 252 len 40960 data: 12 14 17 15 17 18 20 17 10 11 05/30 Write block 66 len 40960 data: 9 11 9 8 8 9 8 8 9 9 06/30 Write block 159 len 24576 data: 9 9 9 8 8 8 07/30 Flush 08/30 Write block 157 len 24576 data: 10 8 10 10 10 9 09/30 Flush 10/30 Flush 11/30 Write block 481 len 40960 data: 12 13 11 9 11 8 10 13 11 9 12/30 Read block 287 len 24576 13/30 Read block 180 len 20480 14/30 Write block 328 len 32768 data: 12 12 12 11 9 8 8 8 15/30 Read block 420 len 12288 16/30 Write block 430 len 12288 data: 13 11 12 17/30 Flush 18/30 Write block 369 len 16384 data: 12 13 11 13 19/30 Read block 131 len 8192 20/30 Write block 141 len 28672 data: 14 13 12 16 15 14 16 21/30 Write block 206 len 20480 data: 14 19 17 15 12 22/30 Read block 421 len 36864 23/30 Write block 135 len 16384 data: 13 11 9 11 24/30 Write block 206 len 8192 data: 15 20 25/30 Read block 301 len 20480 26/30 Read block 579 len 12288 27/30 Write block 17 len 4096 data: 11 28/30 Read block 260 len 20480 29/30 Write block 182 len 12288 data: 7 6 7 30/30 Read block 197 len 32768 ---------------------------------------------------------------- Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Sent Done Done false 1001 Acked Read 2 Sent Done Done false 1002 Acked Read 6 Sent Done Done false 1003 Acked Write 10 Sent Done Done false 1004 Acked Write 10 Sent Done Done false 1005 Acked Write 6 Sent Done Done false 1006 Acked Flush 0 Sent Done Done false 1007 Acked Write 6 Sent Done Done false 1008 Acked Flush 0 Sent Done Done false 1009 Acked Flush 0 Sent Done Done false 1010 Acked Write 10 Sent Done Done false 1011 Acked Read 6 Sent Done Done false 1012 Acked Read 5 Sent Done Done false 1013 Acked Write 8 Sent Done Done false 1014 Acked Read 3 Sent Done Done false 1015 Acked Write 3 Sent Done Done false 1016 Acked Flush 0 Sent Done Done false 1017 Acked Write 4 Sent Done Done false 1018 Acked Read 2 Sent Done Done false 1019 Acked Write 7 Sent Done Done false 1020 Acked Write 5 Sent Done Done false 1021 Acked Read 9 Sent Done Done false 1022 Acked Write 4 Sent Done Done false 1023 Acked Write 2 Sent Done Done false 1024 Acked Read 5 Sent Done Done false 1025 Acked Read 3 Sent Done Done false 1026 Acked Write 1 Sent Done Done false 1027 Acked Read 5 Sent Done Done false 1028 Acked Write 3 Sent Done Done false 1029 Acked Read 8 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 1 0 31 Done 0 29 30 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:58:42.053818824Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1728} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:42.054040522Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1728,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 147 147 147 257 257 257 F T T <--- 1 020-039 138 138 138 241 241 241 F F F 2 040-059 144 144 144 250 250 250 F F F 3 060-079 147 148 148 257 258 258 F F F <--- 4 080-099 135 135 135 237 237 237 F F F 5 100-119 136 148 148 238 258 258 F F F <--- 6 120-139 147 147 147 257 257 257 F T T <--- 7 140-159 142 148 148 248 259 259 F T T <--- 8 160-179 146 148 148 255 259 259 F F F <--- 9 180-199 146 146 146 255 255 255 F T T <--- 10 200-219 140 140 140 244 244 244 F T T <--- 11 220-239 142 142 142 247 247 247 F F F 12 240-259 146 148 148 255 258 258 F F F <--- 13 260-279 142 148 148 247 258 258 F F F <--- 14 280-299 144 144 144 250 250 250 F F F 15 300-319 145 145 145 251 251 251 F F F 16 320-339 142 148 148 248 261 261 F F F <--- 17 340-359 143 143 143 249 249 249 F F F 18 360-379 146 146 146 253 253 253 F T T <--- 19 380-399 146 146 146 253 253 253 F F F 20 400-419 142 142 142 247 247 247 F F F 21 420-439 146 148 148 255 261 261 F F F <--- 22 440-459 136 136 136 239 239 239 F F F 23 460-479 142 142 142 247 247 247 F F F 24 480-499 145 148 148 251 261 261 F F F <--- 25 500-519 146 146 146 252 252 252 F F F 26 520-539 146 146 146 253 253 253 F F F 27 540-559 144 144 144 250 250 250 F F F 28 560-579 145 145 145 251 251 251 F F F 29 580-599 145 145 145 251 251 251 F F F Max gen: 148, Max flush: 261 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 74 Verifying data now No change after verify Loop: 74 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 149 149 149 262 262 262 F F F 1 020-039 138 138 138 241 241 241 F F F 2 040-059 144 144 144 250 250 250 F F F 3 060-079 148 148 148 258 258 258 F F F 4 080-099 135 135 135 237 237 237 F F F 5 100-119 148 148 148 258 258 258 F F F 6 120-139 149 149 149 262 262 262 F F F 7 140-159 149 149 149 262 262 262 F F F 8 160-179 148 148 148 259 259 259 F F F 9 180-199 149 149 149 262 262 262 F F F 10 200-219 149 149 149 262 262 262 F F F 11 220-239 142 142 142 247 247 247 F F F 12 240-259 148 148 148 258 258 258 F F F 13 260-279 148 148 148 258 258 258 F F F 14 280-299 144 144 144 250 250 250 F F F 15 300-319 145 145 145 251 251 251 F F F 16 320-339 148 148 148 261 261 261 F F F 17 340-359 143 143 143 249 249 249 F F F 18 360-379 149 149 149 262 262 262 F F F 19 380-399 146 146 146 253 253 253 F F F 20 400-419 142 142 142 247 247 247 F F F 21 420-439 148 148 148 261 261 261 F F F 22 440-459 136 136 136 239 239 239 F F F 23 460-479 142 142 142 247 247 247 F F F 24 480-499 148 148 148 261 261 261 F F F 25 500-519 146 146 146 252 252 252 F F F 26 520-539 146 146 146 253 253 253 F F F 27 540-559 144 144 144 250 250 250 F F F 28 560-579 145 145 145 251 251 251 F F F 29 580-599 145 145 145 251 251 251 F F F Max gen: 149, Max flush: 262 Begin loop 75 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:46.311251372Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1736,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:46.319473988Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1736,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:46.31980089Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1736} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:46.319837675Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1736} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:46.319847283Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1736} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:46.319858842Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1736} {"msg":"Crucible e3da6787-85e2-44cf-9361-69df65eb24e2 has session id: c0e3fc80-a215-4d33-9f93-46c4a93fa4bb","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:46.320098996Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1736,"session_id":"c0e3fc80-a215-4d33-9f93-46c4a93fa4bb"} {"msg":"Upstairs opts: Upstairs UUID: e3da6787-85e2-44cf-9361-69df65eb24e2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:46.320122123Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1736,"session_id":"c0e3fc80-a215-4d33-9f93-46c4a93fa4bb"} {"msg":"Crucible stats registered with UUID: e3da6787-85e2-44cf-9361-69df65eb24e2","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:46.320132371Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1736,"session_id":"c0e3fc80-a215-4d33-9f93-46c4a93fa4bb"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:58:46.32014152Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1736,"session_id":"c0e3fc80-a215-4d33-9f93-46c4a93fa4bb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:46.320287278Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1736,"":"io task","client":"0","":"downstairs","session_id":"c0e3fc80-a215-4d33-9f93-46c4a93fa4bb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:46.320317924Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1736,"":"io task","client":"2","":"downstairs","session_id":"c0e3fc80-a215-4d33-9f93-46c4a93fa4bb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:46.320328912Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1736,"":"io task","client":"1","":"downstairs","session_id":"c0e3fc80-a215-4d33-9f93-46c4a93fa4bb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:46.320462972Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1736,"task":"crutest"} {"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:46.32048193Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1736} {"msg":"e3da6787-85e2-44cf-9361-69df65eb24e2 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:46.320506146Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1736,"session_id":"c0e3fc80-a215-4d33-9f93-46c4a93fa4bb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:46.320515445Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1736,"client":"0","":"downstairs","session_id":"c0e3fc80-a215-4d33-9f93-46c4a93fa4bb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:46.320525383Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1736,"client":"1","":"downstairs","session_id":"c0e3fc80-a215-4d33-9f93-46c4a93fa4bb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:46.320534872Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1736,"client":"2","":"downstairs","session_id":"c0e3fc80-a215-4d33-9f93-46c4a93fa4bb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:46.32054459Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1736,"":"io task","client":"2","":"downstairs","session_id":"c0e3fc80-a215-4d33-9f93-46c4a93fa4bb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:46.320554769Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1736,"":"io task","client":"0","":"downstairs","session_id":"c0e3fc80-a215-4d33-9f93-46c4a93fa4bb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:46.320564677Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1736,"":"io task","client":"1","":"downstairs","session_id":"c0e3fc80-a215-4d33-9f93-46c4a93fa4bb"} {"msg":"ds_connection connected from Ok(127.0.0.1:49330)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:46.320590544Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1736,"":"io task","client":"2","":"downstairs","session_id":"c0e3fc80-a215-4d33-9f93-46c4a93fa4bb"} {"msg":"ds_connection connected from Ok(127.0.0.1:64648)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:46.320669922Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1736,"":"io task","client":"0","":"downstairs","session_id":"c0e3fc80-a215-4d33-9f93-46c4a93fa4bb"} {"msg":"ds_connection connected from Ok(127.0.0.1:56151)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:46.320695288Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1736,"":"io task","client":"1","":"downstairs","session_id":"c0e3fc80-a215-4d33-9f93-46c4a93fa4bb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:46.321249266Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1736,"client":"1","":"downstairs","session_id":"c0e3fc80-a215-4d33-9f93-46c4a93fa4bb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:46.321318586Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1736,"client":"0","":"downstairs","session_id":"c0e3fc80-a215-4d33-9f93-46c4a93fa4bb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:46.321415382Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1736,"client":"2","":"downstairs","session_id":"c0e3fc80-a215-4d33-9f93-46c4a93fa4bb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:46.321549232Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1736,"session_id":"c0e3fc80-a215-4d33-9f93-46c4a93fa4bb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:46.321625311Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1736,"session_id":"c0e3fc80-a215-4d33-9f93-46c4a93fa4bb"} {"msg":"[0]R flush_numbers[0..12]: [262, 241, 250, 258, 237, 258, 262, 262, 259, 262, 262, 247]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:46.321687372Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1736,"":"downstairs","":"downstairs","session_id":"c0e3fc80-a215-4d33-9f93-46c4a93fa4bb"} {"msg":"[0]R generation[0..12]: [149, 138, 144, 148, 135, 148, 149, 149, 148, 149, 149, 142]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:46.32169943Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1736,"":"downstairs","":"downstairs","session_id":"c0e3fc80-a215-4d33-9f93-46c4a93fa4bb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:46.321710768Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1736,"":"downstairs","":"downstairs","session_id":"c0e3fc80-a215-4d33-9f93-46c4a93fa4bb"} {"msg":"[1]R flush_numbers[0..12]: [262, 241, 250, 258, 237, 258, 262, 262, 259, 262, 262, 247]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:46.321721427Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1736,"":"downstairs","":"downstairs","session_id":"c0e3fc80-a215-4d33-9f93-46c4a93fa4bb"} {"msg":"[1]R generation[0..12]: [149, 138, 144, 148, 135, 148, 149, 149, 148, 149, 149, 142]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:46.321732195Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1736,"":"downstairs","":"downstairs","session_id":"c0e3fc80-a215-4d33-9f93-46c4a93fa4bb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:46.321743344Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1736,"":"downstairs","":"downstairs","session_id":"c0e3fc80-a215-4d33-9f93-46c4a93fa4bb"} {"msg":"[2]R flush_numbers[0..12]: [262, 241, 250, 258, 237, 258, 262, 262, 259, 262, 262, 247]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:46.321753952Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1736,"":"downstairs","":"downstairs","session_id":"c0e3fc80-a215-4d33-9f93-46c4a93fa4bb"} {"msg":"[2]R generation[0..12]: [149, 138, 144, 148, 135, 148, 149, 149, 148, 149, 149, 142]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:46.32176443Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1736,"":"downstairs","":"downstairs","session_id":"c0e3fc80-a215-4d33-9f93-46c4a93fa4bb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:46.321775339Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1736,"":"downstairs","":"downstairs","session_id":"c0e3fc80-a215-4d33-9f93-46c4a93fa4bb"} {"msg":"Max found gen is 150","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:46.321784817Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1736,"":"downstairs","":"downstairs","session_id":"c0e3fc80-a215-4d33-9f93-46c4a93fa4bb"} {"msg":"Generation requested: 150 >= found:150","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:46.321794606Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1736,"":"downstairs","":"downstairs","session_id":"c0e3fc80-a215-4d33-9f93-46c4a93fa4bb"} {"msg":"Next flush: 263","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:46.321804115Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1736,"":"downstairs","":"downstairs","session_id":"c0e3fc80-a215-4d33-9f93-46c4a93fa4bb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:46.321813533Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1736,"":"downstairs","":"downstairs","session_id":"c0e3fc80-a215-4d33-9f93-46c4a93fa4bb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:46.321825341Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1736,"session_id":"c0e3fc80-a215-4d33-9f93-46c4a93fa4bb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:46.32183408Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1736,"client":"0","":"downstairs","session_id":"c0e3fc80-a215-4d33-9f93-46c4a93fa4bb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:46.321843959Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1736,"client":"1","":"downstairs","session_id":"c0e3fc80-a215-4d33-9f93-46c4a93fa4bb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:46.321853977Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1736,"client":"2","":"downstairs","session_id":"c0e3fc80-a215-4d33-9f93-46c4a93fa4bb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:46.321864166Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1736,"session_id":"c0e3fc80-a215-4d33-9f93-46c4a93fa4bb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:46.321873354Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1736,"session_id":"c0e3fc80-a215-4d33-9f93-46c4a93fa4bb"} {"msg":"e3da6787-85e2-44cf-9361-69df65eb24e2 is now active with session: c0e3fc80-a215-4d33-9f93-46c4a93fa4bb","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:46.321883193Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1736,"session_id":"c0e3fc80-a215-4d33-9f93-46c4a93fa4bb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:46.321892042Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1736,"session_id":"c0e3fc80-a215-4d33-9f93-46c4a93fa4bb"} {"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:46.32190094Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1736} {"msg":"Activated sub_volume e3da6787-85e2-44cf-9361-69df65eb24e2","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:46.321909149Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1736} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:46.321917238Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1736,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:46.322235361Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1736,"task":"crutest"} 01/30 Write block 385 len 12288 data: 11 10 8 02/30 Read block 522 len 32768 03/30 Flush 04/30 Write block 237 len 20480 data: 11 11 10 10 9 05/30 Read block 100 len 12288 06/30 Write block 15 len 8192 data: 11 11 07/30 Write block 200 len 24576 data: 9 6 7 10 12 13 08/30 Read block 313 len 32768 09/30 Read block 117 len 40960 10/30 Write block 288 len 12288 data: 13 14 14 11/30 Write block 141 len 28672 data: 15 14 13 17 16 15 17 12/30 Read block 270 len 8192 13/30 Read block 125 len 28672 14/30 Read block 417 len 32768 15/30 Write block 441 len 32768 data: 12 10 10 12 11 11 11 13 16/30 Read block 105 len 8192 17/30 Read block 36 len 12288 18/30 Write block 200 len 4096 data: 10 19/30 Read block 271 len 4096 20/30 Read block 396 len 24576 21/30 Read block 356 len 36864 22/30 Write block 544 len 12288 data: 7 8 10 23/30 Read block 189 len 12288 24/30 Write block 15 len 32768 data: 12 12 12 11 13 14 13 15 25/30 Write block 435 len 16384 data: 12 11 12 13 26/30 Write block 540 len 12288 data: 8 9 9 27/30 Read block 74 len 4096 28/30 Read block 440 len 16384 29/30 Write block 554 len 16384 data: 14 12 10 12 30/30 Read block 86 len 40960 ---------------------------------------------------------------- Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1003 Acked Write 5 Done Done Sent false 1004 Acked Read 3 Done Done Sent false 1005 Acked Write 2 Done Done Sent false 1006 Acked Write 6 Done Done Sent false 1007 Acked Read 8 Done Done Sent false 1008 Acked Read 10 Done Done Sent false 1009 Acked Write 3 Done Done Sent false 1010 Acked Write 7 Done Done Sent false 1011 Acked Read 2 Done Done Sent false 1012 Acked Read 7 Done Done Sent false 1013 Acked Read 8 Done Done Sent false 1014 Acked Write 8 Done Done Sent false 1015 Acked Read 2 Done Done Sent false 1016 Acked Read 3 Done Done Sent false 1017 Acked Write 1 Done Done Sent false 1018 Acked Read 1 Done Done Sent false 1019 Acked Read 6 Done Done Sent false 1020 Acked Read 9 Done Done Sent false 1021 Acked Write 3 Done Done Sent false 1022 Acked Read 3 Done Done Sent false 1023 Acked Write 8 Done Done Sent false 1024 Acked Write 4 Done Done Sent false 1025 Acked Write 3 Done Done Sent false 1026 Acked Read 1 Done Done Sent false 1027 Acked Read 4 Done Done Sent false 1028 Acked Write 4 Done Done Sent false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 27 28 Done 27 26 0 53 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1002)) Some(JobId(1002)) Some(JobId(1002)) Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:58:46.330821294Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1736} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:46.331007607Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1736,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 149 149 149 262 262 262 T T F <--- 1 020-039 138 138 138 241 241 241 T T F <--- 2 040-059 144 144 144 250 250 250 F F F 3 060-079 148 148 148 258 258 258 F F F 4 080-099 135 135 135 237 237 237 F F F 5 100-119 148 148 148 258 258 258 F F F 6 120-139 149 149 149 262 262 262 F F F 7 140-159 149 149 149 262 262 262 T T F <--- 8 160-179 148 148 148 259 259 259 F F F 9 180-199 149 149 149 262 262 262 F F F 10 200-219 149 149 149 262 262 262 T T F <--- 11 220-239 142 142 142 247 247 247 T T F <--- 12 240-259 148 148 148 258 258 258 T T F <--- 13 260-279 148 148 148 258 258 258 F F F 14 280-299 144 144 144 250 250 250 T T F <--- 15 300-319 145 145 145 251 251 251 F F F 16 320-339 148 148 148 261 261 261 F F F 17 340-359 143 143 143 249 249 249 F F F 18 360-379 149 149 149 262 262 262 F F F 19 380-399 150 150 150 263 263 263 F F F 20 400-419 142 142 142 247 247 247 F F F 21 420-439 148 148 148 261 261 261 T T F <--- 22 440-459 136 136 136 239 239 239 T T F <--- 23 460-479 142 142 142 247 247 247 F F F 24 480-499 148 148 148 261 261 261 F F F 25 500-519 146 146 146 252 252 252 F F F 26 520-539 146 146 146 253 253 253 F F F 27 540-559 144 144 144 250 250 250 T T F <--- 28 560-579 145 145 145 251 251 251 F F F 29 580-599 145 145 145 251 251 251 F F F Max gen: 150, Max flush: 263 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 75 Verifying data now No change after verify Loop: 75 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 264 264 264 F F F 1 020-039 151 151 151 264 264 264 F F F 2 040-059 144 144 144 250 250 250 F F F 3 060-079 148 148 148 258 258 258 F F F 4 080-099 135 135 135 237 237 237 F F F 5 100-119 148 148 148 258 258 258 F F F 6 120-139 149 149 149 262 262 262 F F F 7 140-159 151 151 151 264 264 264 F F F 8 160-179 148 148 148 259 259 259 F F F 9 180-199 149 149 149 262 262 262 F F F 10 200-219 151 151 151 264 264 264 F F F 11 220-239 151 151 151 264 264 264 F F F 12 240-259 151 151 151 264 264 264 F F F 13 260-279 148 148 148 258 258 258 F F F 14 280-299 151 151 151 264 264 264 F F F 15 300-319 145 145 145 251 251 251 F F F 16 320-339 148 148 148 261 261 261 F F F 17 340-359 143 143 143 249 249 249 F F F 18 360-379 149 149 149 262 262 262 F F F 19 380-399 150 150 150 263 263 263 F F F 20 400-419 142 142 142 247 247 247 F F F 21 420-439 151 151 151 264 264 264 F F F 22 440-459 151 151 151 264 264 264 F F F 23 460-479 142 142 142 247 247 247 F F F 24 480-499 148 148 148 261 261 261 F F F 25 500-519 146 146 146 252 252 252 F F F 26 520-539 146 146 146 253 253 253 F F F 27 540-559 151 151 151 264 264 264 F F F 28 560-579 145 145 145 251 251 251 F F F 29 580-599 145 145 145 251 251 251 F F F Max gen: 151, Max flush: 264 Begin loop 76 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:50.577674922Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1744,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:50.585770998Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1744,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:50.586078632Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1744} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:50.586102739Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1744} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:50.586114427Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1744} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:50.586126005Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1744} {"msg":"Crucible a0c007d7-2216-4522-b75d-887d9bd77790 has session id: 48d2a5f1-cbc8-4eaf-8271-f8fe02bcca11","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:50.586477813Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1744,"session_id":"48d2a5f1-cbc8-4eaf-8271-f8fe02bcca11"} {"msg":"Upstairs opts: Upstairs UUID: a0c007d7-2216-4522-b75d-887d9bd77790, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:50.586507429Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1744,"session_id":"48d2a5f1-cbc8-4eaf-8271-f8fe02bcca11"} {"msg":"Crucible stats registered with UUID: a0c007d7-2216-4522-b75d-887d9bd77790","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:50.586518627Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1744,"session_id":"48d2a5f1-cbc8-4eaf-8271-f8fe02bcca11"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:58:50.586527806Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1744,"session_id":"48d2a5f1-cbc8-4eaf-8271-f8fe02bcca11"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:50.586655997Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1744,"":"io task","client":"2","":"downstairs","session_id":"48d2a5f1-cbc8-4eaf-8271-f8fe02bcca11"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:50.586672495Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1744,"":"io task","client":"0","":"downstairs","session_id":"48d2a5f1-cbc8-4eaf-8271-f8fe02bcca11"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:50.586682853Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1744,"":"io task","client":"1","":"downstairs","session_id":"48d2a5f1-cbc8-4eaf-8271-f8fe02bcca11"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:50.58684259Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1744,"task":"crutest"} {"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:50.586867086Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1744} {"msg":"a0c007d7-2216-4522-b75d-887d9bd77790 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:50.586880554Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1744,"session_id":"48d2a5f1-cbc8-4eaf-8271-f8fe02bcca11"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:50.586889433Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1744,"client":"0","":"downstairs","session_id":"48d2a5f1-cbc8-4eaf-8271-f8fe02bcca11"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:50.586899341Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1744,"client":"1","":"downstairs","session_id":"48d2a5f1-cbc8-4eaf-8271-f8fe02bcca11"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:50.58690906Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1744,"client":"2","":"downstairs","session_id":"48d2a5f1-cbc8-4eaf-8271-f8fe02bcca11"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:50.586918498Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1744,"":"io task","client":"2","":"downstairs","session_id":"48d2a5f1-cbc8-4eaf-8271-f8fe02bcca11"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:50.586930927Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1744,"":"io task","client":"0","":"downstairs","session_id":"48d2a5f1-cbc8-4eaf-8271-f8fe02bcca11"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:50.586940965Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1744,"":"io task","client":"1","":"downstairs","session_id":"48d2a5f1-cbc8-4eaf-8271-f8fe02bcca11"} {"msg":"ds_connection connected from Ok(127.0.0.1:46492)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:50.587008375Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1744,"":"io task","client":"0","":"downstairs","session_id":"48d2a5f1-cbc8-4eaf-8271-f8fe02bcca11"} {"msg":"ds_connection connected from Ok(127.0.0.1:61392)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:50.587032082Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1744,"":"io task","client":"1","":"downstairs","session_id":"48d2a5f1-cbc8-4eaf-8271-f8fe02bcca11"} {"msg":"ds_connection connected from Ok(127.0.0.1:60351)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:50.58704335Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1744,"":"io task","client":"2","":"downstairs","session_id":"48d2a5f1-cbc8-4eaf-8271-f8fe02bcca11"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:50.58779122Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1744,"client":"0","":"downstairs","session_id":"48d2a5f1-cbc8-4eaf-8271-f8fe02bcca11"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:50.587833253Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1744,"client":"2","":"downstairs","session_id":"48d2a5f1-cbc8-4eaf-8271-f8fe02bcca11"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:50.587851031Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1744,"client":"1","":"downstairs","session_id":"48d2a5f1-cbc8-4eaf-8271-f8fe02bcca11"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:50.588137778Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1744,"session_id":"48d2a5f1-cbc8-4eaf-8271-f8fe02bcca11"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:50.588161815Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1744,"session_id":"48d2a5f1-cbc8-4eaf-8271-f8fe02bcca11"} {"msg":"[0]R flush_numbers[0..12]: [264, 264, 250, 258, 237, 258, 262, 264, 259, 262, 264, 264]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:50.588172613Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1744,"":"downstairs","":"downstairs","session_id":"48d2a5f1-cbc8-4eaf-8271-f8fe02bcca11"} {"msg":"[0]R generation[0..12]: [151, 151, 144, 148, 135, 148, 149, 151, 148, 149, 151, 151]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:50.588183952Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1744,"":"downstairs","":"downstairs","session_id":"48d2a5f1-cbc8-4eaf-8271-f8fe02bcca11"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:50.58819494Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1744,"":"downstairs","":"downstairs","session_id":"48d2a5f1-cbc8-4eaf-8271-f8fe02bcca11"} {"msg":"[1]R flush_numbers[0..12]: [264, 264, 250, 258, 237, 258, 262, 264, 259, 262, 264, 264]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:50.588205888Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1744,"":"downstairs","":"downstairs","session_id":"48d2a5f1-cbc8-4eaf-8271-f8fe02bcca11"} {"msg":"[1]R generation[0..12]: [151, 151, 144, 148, 135, 148, 149, 151, 148, 149, 151, 151]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:50.588218617Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1744,"":"downstairs","":"downstairs","session_id":"48d2a5f1-cbc8-4eaf-8271-f8fe02bcca11"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:50.588229705Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1744,"":"downstairs","":"downstairs","session_id":"48d2a5f1-cbc8-4eaf-8271-f8fe02bcca11"} {"msg":"[2]R flush_numbers[0..12]: [264, 264, 250, 258, 237, 258, 262, 264, 259, 262, 264, 264]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:50.588240413Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1744,"":"downstairs","":"downstairs","session_id":"48d2a5f1-cbc8-4eaf-8271-f8fe02bcca11"} {"msg":"[2]R generation[0..12]: [151, 151, 144, 148, 135, 148, 149, 151, 148, 149, 151, 151]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:50.588250952Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1744,"":"downstairs","":"downstairs","session_id":"48d2a5f1-cbc8-4eaf-8271-f8fe02bcca11"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:50.58826198Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1744,"":"downstairs","":"downstairs","session_id":"48d2a5f1-cbc8-4eaf-8271-f8fe02bcca11"} {"msg":"Max found gen is 152","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:50.588271489Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1744,"":"downstairs","":"downstairs","session_id":"48d2a5f1-cbc8-4eaf-8271-f8fe02bcca11"} {"msg":"Generation requested: 152 >= found:152","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:50.588281247Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1744,"":"downstairs","":"downstairs","session_id":"48d2a5f1-cbc8-4eaf-8271-f8fe02bcca11"} {"msg":"Next flush: 265","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:50.588290576Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1744,"":"downstairs","":"downstairs","session_id":"48d2a5f1-cbc8-4eaf-8271-f8fe02bcca11"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:50.588300115Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1744,"":"downstairs","":"downstairs","session_id":"48d2a5f1-cbc8-4eaf-8271-f8fe02bcca11"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:50.588309863Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1744,"session_id":"48d2a5f1-cbc8-4eaf-8271-f8fe02bcca11"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:50.588318792Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1744,"client":"0","":"downstairs","session_id":"48d2a5f1-cbc8-4eaf-8271-f8fe02bcca11"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:50.58832855Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1744,"client":"1","":"downstairs","session_id":"48d2a5f1-cbc8-4eaf-8271-f8fe02bcca11"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:50.588338199Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1744,"client":"2","":"downstairs","session_id":"48d2a5f1-cbc8-4eaf-8271-f8fe02bcca11"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:50.588348237Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1744,"session_id":"48d2a5f1-cbc8-4eaf-8271-f8fe02bcca11"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:50.588357386Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1744,"session_id":"48d2a5f1-cbc8-4eaf-8271-f8fe02bcca11"} {"msg":"a0c007d7-2216-4522-b75d-887d9bd77790 is now active with session: 48d2a5f1-cbc8-4eaf-8271-f8fe02bcca11","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:50.588368624Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1744,"session_id":"48d2a5f1-cbc8-4eaf-8271-f8fe02bcca11"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:50.588377503Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1744,"session_id":"48d2a5f1-cbc8-4eaf-8271-f8fe02bcca11"} {"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:50.588386542Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1744} {"msg":"Activated sub_volume a0c007d7-2216-4522-b75d-887d9bd77790","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:50.588394641Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1744} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:50.588402759Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1744,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:50.588678149Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1744,"task":"crutest"} 01/30 Write block 540 len 40960 data: 9 10 10 7 8 9 11 10 10 13 02/30 Flush 03/30 Write block 275 len 28672 data: 16 13 17 16 12 13 11 04/30 Read block 148 len 40960 05/30 Flush 06/30 Write block 268 len 4096 data: 12 07/30 Read block 524 len 32768 08/30 Read block 287 len 16384 09/30 Write block 212 len 28672 data: 11 11 11 11 8 6 5 10/30 Read block 239 len 4096 11/30 Write block 39 len 12288 data: 17 16 15 12/30 Read block 387 len 8192 13/30 Read block 85 len 12288 14/30 Write block 285 len 40960 data: 14 16 13 14 15 15 11 10 9 6 15/30 Write block 524 len 20480 data: 8 7 8 10 10 16/30 Flush 17/30 Write block 454 len 4096 data: 13 18/30 Read block 260 len 40960 19/30 Write block 329 len 8192 data: 13 13 20/30 Read block 455 len 4096 21/30 Write block 30 len 32768 data: 10 10 7 5 6 11 11 11 22/30 Flush 23/30 Write block 331 len 12288 data: 12 10 9 24/30 Read block 269 len 24576 25/30 Write block 320 len 4096 data: 11 26/30 Read block 85 len 16384 27/30 Read block 264 len 40960 28/30 Write block 447 len 36864 data: 12 14 14 15 17 16 16 14 9 29/30 Read block 341 len 40960 30/30 Write block 28 len 36864 data: 15 13 11 11 8 6 7 12 12 ---------------------------------------------------------------- Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Sent Done false 1001 Acked Flush 0 Done Sent Done false 1002 Acked Write 7 Done Sent Done false 1003 Acked Read 10 Done Sent Done false 1004 Acked Flush 0 Done Sent Done false 1005 Acked Write 1 Done Sent Done false 1006 Acked Read 8 Done Sent Done false 1007 Acked Read 4 Done Sent Done false 1008 Acked Write 7 Done Sent Done false 1009 Acked Read 1 Done Sent Done false 1010 Acked Write 3 Done Sent Done false 1011 Acked Read 2 Done Sent Done false 1012 Acked Read 3 Done Sent Done false 1013 Acked Write 10 Done Sent Done false 1014 Acked Write 5 Done Sent Done false 1015 Acked Flush 0 Done Sent Done false 1016 Acked Write 1 Done Sent Done false 1017 Acked Read 10 Done Sent Done false 1018 Acked Write 2 Done Sent Done false 1019 Acked Read 1 Done Sent Done false 1020 Acked Write 8 Done Sent Done false 1021 Acked Flush 0 Done Sent Done false 1022 Acked Write 3 Done Sent Done false 1023 Acked Read 6 Done Sent Done false 1024 Acked Write 1 Done Sent Done false 1025 Acked Read 4 Done Sent Done false 1026 Acked Read 10 Done Sent Done false 1027 Acked Write 9 Done Sent Done false 1028 Acked Read 10 Done Sent Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 30 1 32 Done 29 0 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) None Some(JobId(1021)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:58:50.59822726Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1744} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:50.598420091Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1744,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 264 264 264 F F F 1 020-039 152 151 152 268 264 268 T F T <--- 2 040-059 152 144 152 267 250 267 F F F <--- 3 060-079 148 148 148 258 258 258 F F F 4 080-099 135 135 135 237 237 237 F F F 5 100-119 148 148 148 258 258 258 F F F 6 120-139 149 149 149 262 262 262 F F F 7 140-159 151 151 151 264 264 264 F F F 8 160-179 148 148 148 259 259 259 F F F 9 180-199 149 149 149 262 262 262 F F F 10 200-219 152 151 152 267 264 267 F F F <--- 11 220-239 151 151 151 264 264 264 F F F 12 240-259 151 151 151 264 264 264 F F F 13 260-279 152 148 152 267 258 267 F F F <--- 14 280-299 152 151 152 267 264 267 F F F <--- 15 300-319 145 145 145 251 251 251 F F F 16 320-339 152 148 152 268 261 268 T F T <--- 17 340-359 143 143 143 249 249 249 F F F 18 360-379 149 149 149 262 262 262 F F F 19 380-399 150 150 150 263 263 263 F F F 20 400-419 142 142 142 247 247 247 F F F 21 420-439 151 151 151 264 264 264 F F F 22 440-459 152 151 152 268 264 268 T F T <--- 23 460-479 142 142 142 247 247 247 F F F 24 480-499 148 148 148 261 261 261 F F F 25 500-519 146 146 146 252 252 252 F F F 26 520-539 152 146 152 267 253 267 F F F <--- 27 540-559 152 151 152 265 264 265 F F F <--- 28 560-579 145 145 145 251 251 251 F F F 29 580-599 145 145 145 251 251 251 F F F Max gen: 152, Max flush: 268 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 76 Verifying data now No change after verify Loop: 76 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 264 264 264 F F F 1 020-039 153 153 153 269 269 269 F F F 2 040-059 152 152 152 267 267 267 F F F 3 060-079 148 148 148 258 258 258 F F F 4 080-099 135 135 135 237 237 237 F F F 5 100-119 148 148 148 258 258 258 F F F 6 120-139 149 149 149 262 262 262 F F F 7 140-159 151 151 151 264 264 264 F F F 8 160-179 148 148 148 259 259 259 F F F 9 180-199 149 149 149 262 262 262 F F F 10 200-219 152 152 152 267 267 267 F F F 11 220-239 151 151 151 264 264 264 F F F 12 240-259 151 151 151 264 264 264 F F F 13 260-279 152 152 152 267 267 267 F F F 14 280-299 152 152 152 267 267 267 F F F 15 300-319 145 145 145 251 251 251 F F F 16 320-339 153 153 153 269 269 269 F F F 17 340-359 143 143 143 249 249 249 F F F 18 360-379 149 149 149 262 262 262 F F F 19 380-399 150 150 150 263 263 263 F F F 20 400-419 142 142 142 247 247 247 F F F 21 420-439 151 151 151 264 264 264 F F F 22 440-459 153 153 153 269 269 269 F F F 23 460-479 142 142 142 247 247 247 F F F 24 480-499 148 148 148 261 261 261 F F F 25 500-519 146 146 146 252 252 252 F F F 26 520-539 152 152 152 267 267 267 F F F 27 540-559 152 152 152 265 265 265 F F F 28 560-579 145 145 145 251 251 251 F F F 29 580-599 145 145 145 251 251 251 F F F Max gen: 153, Max flush: 269 Begin loop 77 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:54.838208041Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1752,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:54.846233827Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1752,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:54.846521355Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1752} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:54.846545171Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1752} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:54.84655378Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1752} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:54.846565428Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1752} {"msg":"Crucible 5b790a26-d7f5-4b19-b0c5-3771ff38057d has session id: c27b892a-8bff-4376-a475-539a0977fac2","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:54.846949341Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1752,"session_id":"c27b892a-8bff-4376-a475-539a0977fac2"} {"msg":"Upstairs opts: Upstairs UUID: 5b790a26-d7f5-4b19-b0c5-3771ff38057d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:54.846988036Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1752,"session_id":"c27b892a-8bff-4376-a475-539a0977fac2"} {"msg":"Crucible stats registered with UUID: 5b790a26-d7f5-4b19-b0c5-3771ff38057d","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:54.847011442Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1752,"session_id":"c27b892a-8bff-4376-a475-539a0977fac2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:58:54.847023331Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1752,"session_id":"c27b892a-8bff-4376-a475-539a0977fac2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:54.847156771Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1752,"":"io task","client":"1","":"downstairs","session_id":"c27b892a-8bff-4376-a475-539a0977fac2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:54.847187456Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1752,"":"io task","client":"0","":"downstairs","session_id":"c27b892a-8bff-4376-a475-539a0977fac2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:54.847198465Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1752,"":"io task","client":"2","":"downstairs","session_id":"c27b892a-8bff-4376-a475-539a0977fac2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:54.847324576Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1752,"task":"crutest"} {"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:54.847339024Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1752} {"msg":"5b790a26-d7f5-4b19-b0c5-3771ff38057d active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:54.847371049Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1752,"session_id":"c27b892a-8bff-4376-a475-539a0977fac2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:54.847380478Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1752,"client":"0","":"downstairs","session_id":"c27b892a-8bff-4376-a475-539a0977fac2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:54.847390186Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1752,"client":"1","":"downstairs","session_id":"c27b892a-8bff-4376-a475-539a0977fac2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:54.847399895Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1752,"client":"2","":"downstairs","session_id":"c27b892a-8bff-4376-a475-539a0977fac2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:54.847409504Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1752,"":"io task","client":"2","":"downstairs","session_id":"c27b892a-8bff-4376-a475-539a0977fac2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:54.847419412Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1752,"":"io task","client":"0","":"downstairs","session_id":"c27b892a-8bff-4376-a475-539a0977fac2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:54.847429521Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1752,"":"io task","client":"1","":"downstairs","session_id":"c27b892a-8bff-4376-a475-539a0977fac2"} {"msg":"ds_connection connected from Ok(127.0.0.1:59466)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:54.847466645Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1752,"":"io task","client":"2","":"downstairs","session_id":"c27b892a-8bff-4376-a475-539a0977fac2"} {"msg":"ds_connection connected from Ok(127.0.0.1:56065)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:54.847483543Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1752,"":"io task","client":"1","":"downstairs","session_id":"c27b892a-8bff-4376-a475-539a0977fac2"} {"msg":"ds_connection connected from Ok(127.0.0.1:43706)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:54.847494631Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1752,"":"io task","client":"0","":"downstairs","session_id":"c27b892a-8bff-4376-a475-539a0977fac2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:54.848214835Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1752,"client":"0","":"downstairs","session_id":"c27b892a-8bff-4376-a475-539a0977fac2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:54.848241811Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1752,"client":"1","":"downstairs","session_id":"c27b892a-8bff-4376-a475-539a0977fac2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:54.848289554Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1752,"client":"2","":"downstairs","session_id":"c27b892a-8bff-4376-a475-539a0977fac2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:54.848551435Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1752,"session_id":"c27b892a-8bff-4376-a475-539a0977fac2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:54.848561004Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1752,"session_id":"c27b892a-8bff-4376-a475-539a0977fac2"} {"msg":"[0]R flush_numbers[0..12]: [264, 269, 267, 258, 237, 258, 262, 264, 259, 262, 267, 264]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:54.848609597Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1752,"":"downstairs","":"downstairs","session_id":"c27b892a-8bff-4376-a475-539a0977fac2"} {"msg":"[0]R generation[0..12]: [151, 153, 152, 148, 135, 148, 149, 151, 148, 149, 152, 151]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:54.848620905Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1752,"":"downstairs","":"downstairs","session_id":"c27b892a-8bff-4376-a475-539a0977fac2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:54.848631933Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1752,"":"downstairs","":"downstairs","session_id":"c27b892a-8bff-4376-a475-539a0977fac2"} {"msg":"[1]R flush_numbers[0..12]: [264, 269, 267, 258, 237, 258, 262, 264, 259, 262, 267, 264]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:54.848642602Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1752,"":"downstairs","":"downstairs","session_id":"c27b892a-8bff-4376-a475-539a0977fac2"} {"msg":"[1]R generation[0..12]: [151, 153, 152, 148, 135, 148, 149, 151, 148, 149, 152, 151]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:54.84865324Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1752,"":"downstairs","":"downstairs","session_id":"c27b892a-8bff-4376-a475-539a0977fac2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:54.848664478Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1752,"":"downstairs","":"downstairs","session_id":"c27b892a-8bff-4376-a475-539a0977fac2"} {"msg":"[2]R flush_numbers[0..12]: [264, 269, 267, 258, 237, 258, 262, 264, 259, 262, 267, 264]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:54.848675027Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1752,"":"downstairs","":"downstairs","session_id":"c27b892a-8bff-4376-a475-539a0977fac2"} {"msg":"[2]R generation[0..12]: [151, 153, 152, 148, 135, 148, 149, 151, 148, 149, 152, 151]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:54.848685645Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1752,"":"downstairs","":"downstairs","session_id":"c27b892a-8bff-4376-a475-539a0977fac2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:54.848696504Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1752,"":"downstairs","":"downstairs","session_id":"c27b892a-8bff-4376-a475-539a0977fac2"} {"msg":"Max found gen is 154","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:54.848708402Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1752,"":"downstairs","":"downstairs","session_id":"c27b892a-8bff-4376-a475-539a0977fac2"} {"msg":"Generation requested: 154 >= found:154","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:54.848718161Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1752,"":"downstairs","":"downstairs","session_id":"c27b892a-8bff-4376-a475-539a0977fac2"} {"msg":"Next flush: 270","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:54.848727549Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1752,"":"downstairs","":"downstairs","session_id":"c27b892a-8bff-4376-a475-539a0977fac2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:54.848737068Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1752,"":"downstairs","":"downstairs","session_id":"c27b892a-8bff-4376-a475-539a0977fac2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:54.848747046Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1752,"session_id":"c27b892a-8bff-4376-a475-539a0977fac2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:54.848755955Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1752,"client":"0","":"downstairs","session_id":"c27b892a-8bff-4376-a475-539a0977fac2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:54.848765823Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1752,"client":"1","":"downstairs","session_id":"c27b892a-8bff-4376-a475-539a0977fac2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:54.848775652Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1752,"client":"2","":"downstairs","session_id":"c27b892a-8bff-4376-a475-539a0977fac2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:54.848785621Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1752,"session_id":"c27b892a-8bff-4376-a475-539a0977fac2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:54.848794849Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1752,"session_id":"c27b892a-8bff-4376-a475-539a0977fac2"} {"msg":"5b790a26-d7f5-4b19-b0c5-3771ff38057d is now active with session: c27b892a-8bff-4376-a475-539a0977fac2","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:54.848804718Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1752,"session_id":"c27b892a-8bff-4376-a475-539a0977fac2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:54.848813546Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1752,"session_id":"c27b892a-8bff-4376-a475-539a0977fac2"} {"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:54.848822685Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1752} {"msg":"Activated sub_volume 5b790a26-d7f5-4b19-b0c5-3771ff38057d","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:54.848830504Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1752} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:54.848838553Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1752,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:54.849168014Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1752,"task":"crutest"} 01/30 Write block 577 len 32768 data: 11 13 11 10 13 13 14 16 02/30 Read block 150 len 4096 03/30 Read block 111 len 36864 04/30 Flush 05/30 Write block 26 len 20480 data: 15 13 16 14 12 06/30 Read block 444 len 20480 07/30 Read block 351 len 28672 08/30 Read block 528 len 28672 09/30 Read block 396 len 12288 10/30 Flush 11/30 Write block 430 len 16384 data: 14 12 13 13 12/30 Read block 14 len 32768 13/30 Read block 152 len 24576 14/30 Read block 392 len 24576 15/30 Read block 29 len 12288 16/30 Read block 528 len 28672 17/30 Read block 56 len 20480 18/30 Read block 105 len 4096 19/30 Write block 335 len 32768 data: 9 9 11 12 13 12 11 10 20/30 Read block 419 len 36864 21/30 Flush 22/30 Write block 107 len 4096 data: 11 23/30 Write block 533 len 4096 data: 13 24/30 Read block 475 len 28672 25/30 Read block 514 len 32768 26/30 Read block 138 len 32768 27/30 Read block 212 len 24576 28/30 Read block 282 len 32768 29/30 Read block 544 len 40960 30/30 Read block 418 len 16384 ---------------------------------------------------------------- Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Sent Done false 1001 Acked Read 1 Done Sent Done false 1002 Acked Read 9 Done Sent Done false 1003 Acked Flush 0 Done Sent Done false 1004 Acked Write 5 Done Sent Done false 1005 Acked Read 5 Done Sent Done false 1006 Acked Read 7 Done Sent Done false 1007 Acked Read 7 Done Sent Done false 1008 Acked Read 3 Done Sent Done false 1009 Acked Flush 0 Done Sent Done false 1010 Acked Write 4 Done Sent Done false 1011 Acked Read 8 Done Sent Done false 1012 Acked Read 6 Done Sent Done false 1013 Acked Read 6 Done Sent Done false 1014 Acked Read 3 Done Sent Done false 1015 Acked Read 7 Done Sent Done false 1016 Acked Read 5 Done Sent Done false 1017 Acked Read 1 Done Sent Done false 1018 Acked Write 8 Done Sent Done false 1019 Acked Read 9 Done Sent Done false 1020 Acked Flush 0 Done Sent Done false 1021 Acked Write 1 Done Sent Done false 1022 Acked Write 1 Done Sent Done false 1023 Acked Read 7 Done Sent Done false 1024 Acked Read 8 Done Sent Done false 1025 Acked Read 8 Done Sent Done false 1026 Acked Read 6 Done Sent Done false 1027 Acked Read 8 Done Sent Done false 1028 Acked Read 10 Done Sent Done false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 30 1 31 Done 30 0 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) None Some(JobId(1020)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:58:54.858595393Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1752} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:54.858783725Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1752,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 264 264 264 F F F 1 020-039 154 153 154 271 269 271 F F F <--- 2 040-059 152 152 152 267 267 267 F F F 3 060-079 148 148 148 258 258 258 F F F 4 080-099 135 135 135 237 237 237 F F F 5 100-119 148 148 148 258 258 258 T F T <--- 6 120-139 149 149 149 262 262 262 F F F 7 140-159 151 151 151 264 264 264 F F F 8 160-179 148 148 148 259 259 259 F F F 9 180-199 149 149 149 262 262 262 F F F 10 200-219 152 152 152 267 267 267 F F F 11 220-239 151 151 151 264 264 264 F F F 12 240-259 151 151 151 264 264 264 F F F 13 260-279 152 152 152 267 267 267 F F F 14 280-299 152 152 152 267 267 267 F F F 15 300-319 145 145 145 251 251 251 F F F 16 320-339 154 153 154 272 269 272 F F F <--- 17 340-359 154 143 154 272 249 272 F F F <--- 18 360-379 149 149 149 262 262 262 F F F 19 380-399 150 150 150 263 263 263 F F F 20 400-419 142 142 142 247 247 247 F F F 21 420-439 154 151 154 272 264 272 F F F <--- 22 440-459 153 153 153 269 269 269 F F F 23 460-479 142 142 142 247 247 247 F F F 24 480-499 148 148 148 261 261 261 F F F 25 500-519 146 146 146 252 252 252 F F F 26 520-539 152 152 152 267 267 267 T F T <--- 27 540-559 152 152 152 265 265 265 F F F 28 560-579 154 145 154 270 251 270 F F F <--- 29 580-599 154 145 154 270 251 270 F F F <--- Max gen: 154, Max flush: 272 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 77 Verifying data now No change after verify Loop: 77 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 264 264 264 F F F 1 020-039 154 154 154 271 271 271 F F F 2 040-059 152 152 152 267 267 267 F F F 3 060-079 148 148 148 258 258 258 F F F 4 080-099 135 135 135 237 237 237 F F F 5 100-119 155 155 155 273 273 273 F F F 6 120-139 149 149 149 262 262 262 F F F 7 140-159 151 151 151 264 264 264 F F F 8 160-179 148 148 148 259 259 259 F F F 9 180-199 149 149 149 262 262 262 F F F 10 200-219 152 152 152 267 267 267 F F F 11 220-239 151 151 151 264 264 264 F F F 12 240-259 151 151 151 264 264 264 F F F 13 260-279 152 152 152 267 267 267 F F F 14 280-299 152 152 152 267 267 267 F F F 15 300-319 145 145 145 251 251 251 F F F 16 320-339 154 154 154 272 272 272 F F F 17 340-359 154 154 154 272 272 272 F F F 18 360-379 149 149 149 262 262 262 F F F 19 380-399 150 150 150 263 263 263 F F F 20 400-419 142 142 142 247 247 247 F F F 21 420-439 154 154 154 272 272 272 F F F 22 440-459 153 153 153 269 269 269 F F F 23 460-479 142 142 142 247 247 247 F F F 24 480-499 148 148 148 261 261 261 F F F 25 500-519 146 146 146 252 252 252 F F F 26 520-539 155 155 155 273 273 273 F F F 27 540-559 152 152 152 265 265 265 F F F 28 560-579 154 154 154 270 270 270 F F F 29 580-599 154 154 154 270 270 270 F F F Max gen: 155, Max flush: 273 Begin loop 78 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:59.097482476Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1760,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:59.105563984Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1760,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:59.105846392Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1760} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:59.105869529Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1760} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:59.105878298Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1760} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:59.105889826Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1760} {"msg":"Crucible e1118300-0041-40ae-8687-a8da95162aab has session id: 7d3fc038-cecc-49ad-bdbd-bc4fb3a363f6","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:59.106246113Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1760,"session_id":"7d3fc038-cecc-49ad-bdbd-bc4fb3a363f6"} {"msg":"Upstairs opts: Upstairs UUID: e1118300-0041-40ae-8687-a8da95162aab, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:59.106291027Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1760,"session_id":"7d3fc038-cecc-49ad-bdbd-bc4fb3a363f6"} {"msg":"Crucible stats registered with UUID: e1118300-0041-40ae-8687-a8da95162aab","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:59.106301655Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1760,"session_id":"7d3fc038-cecc-49ad-bdbd-bc4fb3a363f6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:58:59.106310774Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1760,"session_id":"7d3fc038-cecc-49ad-bdbd-bc4fb3a363f6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:59.106455543Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1760,"":"io task","client":"0","":"downstairs","session_id":"7d3fc038-cecc-49ad-bdbd-bc4fb3a363f6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:59.10647205Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1760,"":"io task","client":"1","":"downstairs","session_id":"7d3fc038-cecc-49ad-bdbd-bc4fb3a363f6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:59.106482349Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1760,"":"io task","client":"2","":"downstairs","session_id":"7d3fc038-cecc-49ad-bdbd-bc4fb3a363f6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:59.106634266Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1760,"task":"crutest"} {"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:59.106649154Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1760} {"msg":"e1118300-0041-40ae-8687-a8da95162aab active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:59.106681419Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1760,"session_id":"7d3fc038-cecc-49ad-bdbd-bc4fb3a363f6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:59.106692788Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1760,"client":"0","":"downstairs","session_id":"7d3fc038-cecc-49ad-bdbd-bc4fb3a363f6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:59.106702736Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1760,"client":"1","":"downstairs","session_id":"7d3fc038-cecc-49ad-bdbd-bc4fb3a363f6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:59.106712645Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1760,"client":"2","":"downstairs","session_id":"7d3fc038-cecc-49ad-bdbd-bc4fb3a363f6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:59.106722263Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1760,"":"io task","client":"2","":"downstairs","session_id":"7d3fc038-cecc-49ad-bdbd-bc4fb3a363f6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:59.106732262Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1760,"":"io task","client":"0","":"downstairs","session_id":"7d3fc038-cecc-49ad-bdbd-bc4fb3a363f6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:59.106764577Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1760,"":"io task","client":"1","":"downstairs","session_id":"7d3fc038-cecc-49ad-bdbd-bc4fb3a363f6"} {"msg":"ds_connection connected from Ok(127.0.0.1:60905)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:59.106787004Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1760,"":"io task","client":"2","":"downstairs","session_id":"7d3fc038-cecc-49ad-bdbd-bc4fb3a363f6"} {"msg":"ds_connection connected from Ok(127.0.0.1:59556)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:59.106798342Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1760,"":"io task","client":"1","":"downstairs","session_id":"7d3fc038-cecc-49ad-bdbd-bc4fb3a363f6"} {"msg":"ds_connection connected from Ok(127.0.0.1:32906)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:59.106871831Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1760,"":"io task","client":"0","":"downstairs","session_id":"7d3fc038-cecc-49ad-bdbd-bc4fb3a363f6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:59.107468103Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1760,"client":"2","":"downstairs","session_id":"7d3fc038-cecc-49ad-bdbd-bc4fb3a363f6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:59.107528044Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1760,"client":"0","":"downstairs","session_id":"7d3fc038-cecc-49ad-bdbd-bc4fb3a363f6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:59.107552051Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1760,"client":"1","":"downstairs","session_id":"7d3fc038-cecc-49ad-bdbd-bc4fb3a363f6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:59.107787616Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1760,"session_id":"7d3fc038-cecc-49ad-bdbd-bc4fb3a363f6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:59.10782561Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1760,"session_id":"7d3fc038-cecc-49ad-bdbd-bc4fb3a363f6"} {"msg":"[0]R flush_numbers[0..12]: [264, 271, 267, 258, 237, 273, 262, 264, 259, 262, 267, 264]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:59.107866874Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1760,"":"downstairs","":"downstairs","session_id":"7d3fc038-cecc-49ad-bdbd-bc4fb3a363f6"} {"msg":"[0]R generation[0..12]: [151, 154, 152, 148, 135, 155, 149, 151, 148, 149, 152, 151]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:59.107881382Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1760,"":"downstairs","":"downstairs","session_id":"7d3fc038-cecc-49ad-bdbd-bc4fb3a363f6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:59.107892671Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1760,"":"downstairs","":"downstairs","session_id":"7d3fc038-cecc-49ad-bdbd-bc4fb3a363f6"} {"msg":"[1]R flush_numbers[0..12]: [264, 271, 267, 258, 237, 273, 262, 264, 259, 262, 267, 264]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:59.107903449Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1760,"":"downstairs","":"downstairs","session_id":"7d3fc038-cecc-49ad-bdbd-bc4fb3a363f6"} {"msg":"[1]R generation[0..12]: [151, 154, 152, 148, 135, 155, 149, 151, 148, 149, 152, 151]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:59.107914167Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1760,"":"downstairs","":"downstairs","session_id":"7d3fc038-cecc-49ad-bdbd-bc4fb3a363f6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:59.107925206Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1760,"":"downstairs","":"downstairs","session_id":"7d3fc038-cecc-49ad-bdbd-bc4fb3a363f6"} {"msg":"[2]R flush_numbers[0..12]: [264, 271, 267, 258, 237, 273, 262, 264, 259, 262, 267, 264]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:59.107935954Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1760,"":"downstairs","":"downstairs","session_id":"7d3fc038-cecc-49ad-bdbd-bc4fb3a363f6"} {"msg":"[2]R generation[0..12]: [151, 154, 152, 148, 135, 155, 149, 151, 148, 149, 152, 151]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:59.107946573Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1760,"":"downstairs","":"downstairs","session_id":"7d3fc038-cecc-49ad-bdbd-bc4fb3a363f6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:59.107957491Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1760,"":"downstairs","":"downstairs","session_id":"7d3fc038-cecc-49ad-bdbd-bc4fb3a363f6"} {"msg":"Max found gen is 156","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:59.10796724Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1760,"":"downstairs","":"downstairs","session_id":"7d3fc038-cecc-49ad-bdbd-bc4fb3a363f6"} {"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:59.107976988Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1760,"":"downstairs","":"downstairs","session_id":"7d3fc038-cecc-49ad-bdbd-bc4fb3a363f6"} {"msg":"Next flush: 274","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:59.107986437Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1760,"":"downstairs","":"downstairs","session_id":"7d3fc038-cecc-49ad-bdbd-bc4fb3a363f6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:59.107995995Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1760,"":"downstairs","":"downstairs","session_id":"7d3fc038-cecc-49ad-bdbd-bc4fb3a363f6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:59.108017372Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1760,"session_id":"7d3fc038-cecc-49ad-bdbd-bc4fb3a363f6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:59.108026251Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1760,"client":"0","":"downstairs","session_id":"7d3fc038-cecc-49ad-bdbd-bc4fb3a363f6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:59.108037429Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1760,"client":"1","":"downstairs","session_id":"7d3fc038-cecc-49ad-bdbd-bc4fb3a363f6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:59.108047418Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1760,"client":"2","":"downstairs","session_id":"7d3fc038-cecc-49ad-bdbd-bc4fb3a363f6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:59.108057646Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1760,"session_id":"7d3fc038-cecc-49ad-bdbd-bc4fb3a363f6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:59.108066855Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1760,"session_id":"7d3fc038-cecc-49ad-bdbd-bc4fb3a363f6"} {"msg":"e1118300-0041-40ae-8687-a8da95162aab is now active with session: 7d3fc038-cecc-49ad-bdbd-bc4fb3a363f6","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:59.108076993Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1760,"session_id":"7d3fc038-cecc-49ad-bdbd-bc4fb3a363f6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:59.108085632Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1760,"session_id":"7d3fc038-cecc-49ad-bdbd-bc4fb3a363f6"} {"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:59.108094681Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1760} {"msg":"Activated sub_volume e1118300-0041-40ae-8687-a8da95162aab","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:59.10810283Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1760} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:59.108110848Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1760,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:59.108406245Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1760,"task":"crutest"} 01/30 Write block 405 len 32768 data: 11 8 9 11 11 14 19 16 02/30 Flush 03/30 Write block 73 len 16384 data: 9 10 10 9 04/30 Write block 472 len 20480 data: 12 8 11 13 11 05/30 Write block 474 len 32768 data: 12 14 12 12 14 15 14 13 06/30 Read block 535 len 36864 07/30 Write block 588 len 12288 data: 12 11 10 08/30 Write block 386 len 12288 data: 11 9 8 09/30 Read block 191 len 16384 10/30 Write block 67 len 12288 data: 12 10 9 11/30 Write block 150 len 36864 data: 12 13 13 14 10 5 8 11 9 12/30 Write block 440 len 16384 data: 8 13 11 11 13/30 Write block 53 len 36864 data: 10 13 12 14 10 9 6 9 8 14/30 Write block 509 len 36864 data: 6 12 11 12 16 16 15 14 13 15/30 Write block 222 len 24576 data: 10 9 10 7 6 7 16/30 Read block 1 len 12288 17/30 Read block 415 len 4096 18/30 Write block 435 len 24576 data: 13 12 13 14 11 9 19/30 Read block 67 len 32768 20/30 Read block 111 len 36864 21/30 Write block 590 len 24576 data: 11 13 14 14 14 13 22/30 Write block 289 len 8192 data: 16 16 23/30 Read block 589 len 40960 24/30 Read block 154 len 4096 25/30 Read block 500 len 16384 26/30 Write block 368 len 40960 data: 14 13 14 12 14 11 10 10 10 10 27/30 Write block 444 len 20480 data: 13 12 12 13 15 28/30 Write block 488 len 20480 data: 14 12 10 6 7 29/30 Read block 522 len 12288 30/30 Write block 48 len 8192 data: 17 16 ---------------------------------------------------------------- Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Flush 0 Done Sent Done false 1002 Acked Write 4 Done Sent Done false 1003 Acked Write 5 Done Sent Done false 1004 Acked Write 8 Done Sent Done false 1005 Acked Read 9 Done Sent Done false 1006 Acked Write 3 Done Sent Done false 1007 Acked Write 3 Done Sent Done false 1008 Acked Read 4 Done Sent Done false 1009 Acked Write 3 Done Sent Done false 1010 Acked Write 9 Done Sent Done false 1011 Acked Write 4 Done Sent Done false 1012 Acked Write 9 Done Sent Done false 1013 Acked Write 9 Done Sent Done false 1014 Acked Write 6 Done Sent Done false 1015 Acked Read 3 Done Sent Done false 1016 Acked Read 1 Done Sent Done false 1017 Acked Write 6 Done Sent Done false 1018 Acked Read 8 Done Sent Done false 1019 Acked Read 9 Done Sent Done false 1020 Acked Write 6 Done Sent Done false 1021 Acked Write 2 Done Sent Done false 1022 Acked Read 10 Done Sent Done false 1023 Acked Read 1 Done Sent Done false 1024 Acked Read 4 Done Sent Done false 1025 Acked Write 10 Done Sent Done false 1026 Acked Write 5 Done Sent Done false 1027 Acked Write 5 Done Sent Done false 1028 Acked Read 3 Done Sent Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 29 1 31 Done 29 1 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1001)) None Some(JobId(1001)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:58:59.116126966Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1760} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:58:59.116318317Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1760,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 264 264 264 F F F 1 020-039 154 154 154 271 271 271 F F F 2 040-059 152 152 152 267 267 267 T F T <--- 3 060-079 148 148 148 258 258 258 T F T <--- 4 080-099 135 135 135 237 237 237 F F F 5 100-119 155 155 155 273 273 273 F F F 6 120-139 149 149 149 262 262 262 F F F 7 140-159 151 151 151 264 264 264 T F T <--- 8 160-179 148 148 148 259 259 259 F F F 9 180-199 149 149 149 262 262 262 F F F 10 200-219 152 152 152 267 267 267 F F F 11 220-239 151 151 151 264 264 264 T F T <--- 12 240-259 151 151 151 264 264 264 F F F 13 260-279 152 152 152 267 267 267 F F F 14 280-299 152 152 152 267 267 267 T F T <--- 15 300-319 145 145 145 251 251 251 F F F 16 320-339 154 154 154 272 272 272 F F F 17 340-359 154 154 154 272 272 272 F F F 18 360-379 149 149 149 262 262 262 T F T <--- 19 380-399 150 150 150 263 263 263 T F T <--- 20 400-419 156 142 156 274 247 274 F T F <--- 21 420-439 154 154 154 272 272 272 T F T <--- 22 440-459 153 153 153 269 269 269 T F T <--- 23 460-479 142 142 142 247 247 247 T F T <--- 24 480-499 148 148 148 261 261 261 T F T <--- 25 500-519 146 146 146 252 252 252 T F T <--- 26 520-539 155 155 155 273 273 273 F F F 27 540-559 152 152 152 265 265 265 F F F 28 560-579 154 154 154 270 270 270 F F F 29 580-599 154 154 154 270 270 270 T F T <--- Max gen: 156, Max flush: 274 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 78 Verifying data now No change after verify Loop: 78 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 264 264 264 F F F 1 020-039 154 154 154 271 271 271 F F F 2 040-059 157 157 157 275 275 275 F F F 3 060-079 157 157 157 275 275 275 F F F 4 080-099 135 135 135 237 237 237 F F F 5 100-119 155 155 155 273 273 273 F F F 6 120-139 149 149 149 262 262 262 F F F 7 140-159 157 157 157 275 275 275 F F F 8 160-179 148 148 148 259 259 259 F F F 9 180-199 149 149 149 262 262 262 F F F 10 200-219 152 152 152 267 267 267 F F F 11 220-239 157 157 157 275 275 275 F F F 12 240-259 151 151 151 264 264 264 F F F 13 260-279 152 152 152 267 267 267 F F F 14 280-299 157 157 157 275 275 275 F F F 15 300-319 145 145 145 251 251 251 F F F 16 320-339 154 154 154 272 272 272 F F F 17 340-359 154 154 154 272 272 272 F F F 18 360-379 157 157 157 275 275 275 F F F 19 380-399 157 157 157 275 275 275 F F F 20 400-419 156 156 156 274 274 274 F F F 21 420-439 157 157 157 275 275 275 F F F 22 440-459 157 157 157 275 275 275 F F F 23 460-479 157 157 157 275 275 275 F F F 24 480-499 157 157 157 275 275 275 F F F 25 500-519 157 157 157 275 275 275 F F F 26 520-539 155 155 155 273 273 273 F F F 27 540-559 152 152 152 265 265 265 F F F 28 560-579 154 154 154 270 270 270 F F F 29 580-599 157 157 157 275 275 275 F F F Max gen: 157, Max flush: 275 Begin loop 79 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:03.369940517Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1768,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:03.378169803Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1768,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:03.378507603Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1768} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:03.378533619Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1768} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:03.378542148Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1768} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:03.378556176Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1768} {"msg":"Crucible c120d3b1-6a68-450f-a51d-b722d882f085 has session id: b6ac3b94-2f08-42c9-9fb4-1920d54df56e","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:03.378852052Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1768,"session_id":"b6ac3b94-2f08-42c9-9fb4-1920d54df56e"} {"msg":"Upstairs opts: Upstairs UUID: c120d3b1-6a68-450f-a51d-b722d882f085, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:03.378882197Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1768,"session_id":"b6ac3b94-2f08-42c9-9fb4-1920d54df56e"} {"msg":"Crucible stats registered with UUID: c120d3b1-6a68-450f-a51d-b722d882f085","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:03.378893426Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1768,"session_id":"b6ac3b94-2f08-42c9-9fb4-1920d54df56e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:59:03.378902674Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1768,"session_id":"b6ac3b94-2f08-42c9-9fb4-1920d54df56e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:03.379047103Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1768,"":"io task","client":"2","":"downstairs","session_id":"b6ac3b94-2f08-42c9-9fb4-1920d54df56e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:03.379063681Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1768,"":"io task","client":"0","":"downstairs","session_id":"b6ac3b94-2f08-42c9-9fb4-1920d54df56e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:03.379073849Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1768,"":"io task","client":"1","":"downstairs","session_id":"b6ac3b94-2f08-42c9-9fb4-1920d54df56e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:03.379245374Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1768,"task":"crutest"} {"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:03.379260262Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1768} {"msg":"c120d3b1-6a68-450f-a51d-b722d882f085 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:03.379292977Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1768,"session_id":"b6ac3b94-2f08-42c9-9fb4-1920d54df56e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:03.379302505Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1768,"client":"0","":"downstairs","session_id":"b6ac3b94-2f08-42c9-9fb4-1920d54df56e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:03.379312234Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1768,"client":"1","":"downstairs","session_id":"b6ac3b94-2f08-42c9-9fb4-1920d54df56e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:03.379321953Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1768,"client":"2","":"downstairs","session_id":"b6ac3b94-2f08-42c9-9fb4-1920d54df56e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:03.379331691Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1768,"":"io task","client":"2","":"downstairs","session_id":"b6ac3b94-2f08-42c9-9fb4-1920d54df56e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:03.379344149Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1768,"":"io task","client":"0","":"downstairs","session_id":"b6ac3b94-2f08-42c9-9fb4-1920d54df56e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:03.379354228Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1768,"":"io task","client":"1","":"downstairs","session_id":"b6ac3b94-2f08-42c9-9fb4-1920d54df56e"} {"msg":"ds_connection connected from Ok(127.0.0.1:38666)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:03.379420948Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1768,"":"io task","client":"0","":"downstairs","session_id":"b6ac3b94-2f08-42c9-9fb4-1920d54df56e"} {"msg":"ds_connection connected from Ok(127.0.0.1:48400)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:03.379446484Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1768,"":"io task","client":"2","":"downstairs","session_id":"b6ac3b94-2f08-42c9-9fb4-1920d54df56e"} {"msg":"ds_connection connected from Ok(127.0.0.1:44213)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:03.379457973Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1768,"":"io task","client":"1","":"downstairs","session_id":"b6ac3b94-2f08-42c9-9fb4-1920d54df56e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:03.380185215Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1768,"client":"2","":"downstairs","session_id":"b6ac3b94-2f08-42c9-9fb4-1920d54df56e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:03.380206092Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1768,"client":"1","":"downstairs","session_id":"b6ac3b94-2f08-42c9-9fb4-1920d54df56e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:03.380300278Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1768,"client":"0","":"downstairs","session_id":"b6ac3b94-2f08-42c9-9fb4-1920d54df56e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:03.380551071Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1768,"session_id":"b6ac3b94-2f08-42c9-9fb4-1920d54df56e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:03.380611712Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1768,"session_id":"b6ac3b94-2f08-42c9-9fb4-1920d54df56e"} {"msg":"[0]R flush_numbers[0..12]: [264, 271, 275, 275, 237, 273, 262, 275, 259, 262, 267, 275]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:03.380658235Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1768,"":"downstairs","":"downstairs","session_id":"b6ac3b94-2f08-42c9-9fb4-1920d54df56e"} {"msg":"[0]R generation[0..12]: [151, 154, 157, 157, 135, 155, 149, 157, 148, 149, 152, 157]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:03.380669954Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1768,"":"downstairs","":"downstairs","session_id":"b6ac3b94-2f08-42c9-9fb4-1920d54df56e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:03.380681162Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1768,"":"downstairs","":"downstairs","session_id":"b6ac3b94-2f08-42c9-9fb4-1920d54df56e"} {"msg":"[1]R flush_numbers[0..12]: [264, 271, 275, 275, 237, 273, 262, 275, 259, 262, 267, 275]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:03.38069203Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1768,"":"downstairs","":"downstairs","session_id":"b6ac3b94-2f08-42c9-9fb4-1920d54df56e"} {"msg":"[1]R generation[0..12]: [151, 154, 157, 157, 135, 155, 149, 157, 148, 149, 152, 157]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:03.380703079Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1768,"":"downstairs","":"downstairs","session_id":"b6ac3b94-2f08-42c9-9fb4-1920d54df56e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:03.380716777Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1768,"":"downstairs","":"downstairs","session_id":"b6ac3b94-2f08-42c9-9fb4-1920d54df56e"} {"msg":"[2]R flush_numbers[0..12]: [264, 271, 275, 275, 237, 273, 262, 275, 259, 262, 267, 275]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:03.380727665Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1768,"":"downstairs","":"downstairs","session_id":"b6ac3b94-2f08-42c9-9fb4-1920d54df56e"} {"msg":"[2]R generation[0..12]: [151, 154, 157, 157, 135, 155, 149, 157, 148, 149, 152, 157]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:03.380738364Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1768,"":"downstairs","":"downstairs","session_id":"b6ac3b94-2f08-42c9-9fb4-1920d54df56e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:03.380749052Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1768,"":"downstairs","":"downstairs","session_id":"b6ac3b94-2f08-42c9-9fb4-1920d54df56e"} {"msg":"Max found gen is 158","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:03.380758721Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1768,"":"downstairs","":"downstairs","session_id":"b6ac3b94-2f08-42c9-9fb4-1920d54df56e"} {"msg":"Generation requested: 158 >= found:158","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:03.380768489Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1768,"":"downstairs","":"downstairs","session_id":"b6ac3b94-2f08-42c9-9fb4-1920d54df56e"} {"msg":"Next flush: 276","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:03.380777978Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1768,"":"downstairs","":"downstairs","session_id":"b6ac3b94-2f08-42c9-9fb4-1920d54df56e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:03.380787526Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1768,"":"downstairs","":"downstairs","session_id":"b6ac3b94-2f08-42c9-9fb4-1920d54df56e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:03.380797725Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1768,"session_id":"b6ac3b94-2f08-42c9-9fb4-1920d54df56e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:03.380806714Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1768,"client":"0","":"downstairs","session_id":"b6ac3b94-2f08-42c9-9fb4-1920d54df56e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:03.380816952Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1768,"client":"1","":"downstairs","session_id":"b6ac3b94-2f08-42c9-9fb4-1920d54df56e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:03.380826931Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1768,"client":"2","":"downstairs","session_id":"b6ac3b94-2f08-42c9-9fb4-1920d54df56e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:03.380836969Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1768,"session_id":"b6ac3b94-2f08-42c9-9fb4-1920d54df56e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:03.380846208Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1768,"session_id":"b6ac3b94-2f08-42c9-9fb4-1920d54df56e"} {"msg":"c120d3b1-6a68-450f-a51d-b722d882f085 is now active with session: b6ac3b94-2f08-42c9-9fb4-1920d54df56e","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:03.380856166Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1768,"session_id":"b6ac3b94-2f08-42c9-9fb4-1920d54df56e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:03.380866085Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1768,"session_id":"b6ac3b94-2f08-42c9-9fb4-1920d54df56e"} {"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:03.380875293Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1768} {"msg":"Activated sub_volume c120d3b1-6a68-450f-a51d-b722d882f085","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:03.380883272Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1768} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:03.380891181Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1768,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:03.381208744Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1768,"task":"crutest"} 01/30 Write block 453 len 4096 data: 17 02/30 Write block 219 len 32768 data: 5 8 10 11 10 11 8 7 03/30 Write block 192 len 32768 data: 18 18 17 17 13 14 13 12 04/30 Read block 234 len 28672 05/30 Write block 104 len 36864 data: 13 13 10 12 11 8 8 12 12 06/30 Write block 136 len 32768 data: 12 10 12 12 13 16 15 14 07/30 Read block 217 len 40960 08/30 Read block 402 len 36864 09/30 Read block 523 len 28672 10/30 Read block 421 len 32768 11/30 Read block 508 len 4096 12/30 Write block 234 len 16384 data: 7 11 10 12 13/30 Write block 163 len 12288 data: 9 9 10 14/30 Read block 396 len 32768 15/30 Flush 16/30 Write block 557 len 24576 data: 13 11 9 8 9 8 17/30 Read block 328 len 32768 18/30 Write block 289 len 24576 data: 17 17 12 11 10 7 19/30 Flush 20/30 Write block 559 len 4096 data: 10 21/30 Read block 510 len 32768 22/30 Flush 23/30 Write block 127 len 12288 data: 9 7 9 24/30 Flush 25/30 Write block 224 len 32768 data: 12 9 8 8 10 12 13 11 26/30 Read block 28 len 20480 27/30 Write block 503 len 8192 data: 13 12 28/30 Read block 324 len 4096 29/30 Read block 305 len 24576 30/30 Read block 246 len 8192 ---------------------------------------------------------------- Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Sent Done false 1001 Acked Write 8 Done Sent Done false 1002 Acked Write 8 Done Sent Done false 1003 Acked Read 7 Done Sent Done false 1004 Acked Write 9 Done Sent Done false 1005 Acked Write 8 Done Sent Done false 1006 Acked Read 10 Done Sent Done false 1007 Acked Read 9 Done Sent Done false 1008 Acked Read 7 Done Sent Done false 1009 Acked Read 8 Done Sent Done false 1010 Acked Read 1 Done Sent Done false 1011 Acked Write 4 Done Sent Done false 1012 Acked Write 3 Done Sent Done false 1013 Acked Read 8 Done Sent Done false 1014 Acked Flush 0 Done Sent Done false 1015 Acked Write 6 Done Sent Done false 1016 Acked Read 8 Done Sent Done false 1017 Acked Write 6 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Write 1 Done Sent Done false 1020 Acked Read 8 Done Sent Done false 1021 Acked Flush 0 Done Sent Done false 1022 Acked Write 3 Done Sent Done false 1023 Acked Flush 0 Done Sent Done false 1024 Acked Write 8 Done Sent Done false 1025 Acked Read 5 Done Sent Done false 1026 Acked Write 2 Done Sent Done false 1027 Acked Read 1 Done Sent Done false 1028 Acked Read 6 Done Sent Done false 1029 Acked Read 2 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 30 0 30 Done 30 0 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) None Some(JobId(1023)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:59:03.389727977Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1768} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:03.389926138Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1768,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 264 264 264 F F F 1 020-039 154 154 154 271 271 271 F F F 2 040-059 157 157 157 275 275 275 F F F 3 060-079 157 157 157 275 275 275 F F F 4 080-099 135 135 135 237 237 237 F F F 5 100-119 158 155 158 276 273 276 F F F <--- 6 120-139 158 149 158 279 262 279 F F F <--- 7 140-159 158 157 158 276 275 276 F F F <--- 8 160-179 158 148 158 276 259 276 F F F <--- 9 180-199 158 149 158 276 262 276 F F F <--- 10 200-219 158 152 158 276 267 276 F F F <--- 11 220-239 158 157 158 276 275 276 T F T <--- 12 240-259 151 151 151 264 264 264 F F F 13 260-279 152 152 152 267 267 267 F F F 14 280-299 158 157 158 277 275 277 F F F <--- 15 300-319 145 145 145 251 251 251 F F F 16 320-339 154 154 154 272 272 272 F F F 17 340-359 154 154 154 272 272 272 F F F 18 360-379 157 157 157 275 275 275 F F F 19 380-399 157 157 157 275 275 275 F F F 20 400-419 156 156 156 274 274 274 F F F 21 420-439 157 157 157 275 275 275 F F F 22 440-459 158 157 158 276 275 276 F F F <--- 23 460-479 157 157 157 275 275 275 F F F 24 480-499 157 157 157 275 275 275 F F F 25 500-519 157 157 157 275 275 275 T F T <--- 26 520-539 155 155 155 273 273 273 F F F 27 540-559 158 152 158 278 265 278 F F F <--- 28 560-579 158 154 158 277 270 277 F F F <--- 29 580-599 157 157 157 275 275 275 F F F Max gen: 158, Max flush: 279 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 79 Verifying data now No change after verify Loop: 79 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 264 264 264 F F F 1 020-039 154 154 154 271 271 271 F F F 2 040-059 157 157 157 275 275 275 F F F 3 060-079 157 157 157 275 275 275 F F F 4 080-099 135 135 135 237 237 237 F F F 5 100-119 158 158 158 276 276 276 F F F 6 120-139 158 158 158 279 279 279 F F F 7 140-159 158 158 158 276 276 276 F F F 8 160-179 158 158 158 276 276 276 F F F 9 180-199 158 158 158 276 276 276 F F F 10 200-219 158 158 158 276 276 276 F F F 11 220-239 159 159 159 280 280 280 F F F 12 240-259 151 151 151 264 264 264 F F F 13 260-279 152 152 152 267 267 267 F F F 14 280-299 158 158 158 277 277 277 F F F 15 300-319 145 145 145 251 251 251 F F F 16 320-339 154 154 154 272 272 272 F F F 17 340-359 154 154 154 272 272 272 F F F 18 360-379 157 157 157 275 275 275 F F F 19 380-399 157 157 157 275 275 275 F F F 20 400-419 156 156 156 274 274 274 F F F 21 420-439 157 157 157 275 275 275 F F F 22 440-459 158 158 158 276 276 276 F F F 23 460-479 157 157 157 275 275 275 F F F 24 480-499 157 157 157 275 275 275 F F F 25 500-519 159 159 159 280 280 280 F F F 26 520-539 155 155 155 273 273 273 F F F 27 540-559 158 158 158 278 278 278 F F F 28 560-579 158 158 158 277 277 277 F F F 29 580-599 157 157 157 275 275 275 F F F Max gen: 159, Max flush: 280 Begin loop 80 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:07.636427588Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1776,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:59:07.643648902Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1776,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:07.643697185Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1776,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:07.65078417Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1776,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:07.651065318Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1776} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:07.651102483Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1776} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:07.651111721Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1776} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:07.65112325Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1776} {"msg":"Crucible cf9ae119-7675-4d5d-8b6e-1488d28d61aa has session id: f09f36ba-0c9d-4576-bf14-4efeee7f01ec","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:07.651456521Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1776,"session_id":"f09f36ba-0c9d-4576-bf14-4efeee7f01ec"} {"msg":"Upstairs opts: Upstairs UUID: cf9ae119-7675-4d5d-8b6e-1488d28d61aa, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:07.651477228Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1776,"session_id":"f09f36ba-0c9d-4576-bf14-4efeee7f01ec"} {"msg":"Crucible stats registered with UUID: cf9ae119-7675-4d5d-8b6e-1488d28d61aa","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:07.651489236Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1776,"session_id":"f09f36ba-0c9d-4576-bf14-4efeee7f01ec"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:59:07.651498314Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1776,"session_id":"f09f36ba-0c9d-4576-bf14-4efeee7f01ec"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:07.651630425Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1776,"":"io task","client":"2","":"downstairs","session_id":"f09f36ba-0c9d-4576-bf14-4efeee7f01ec"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:07.65166217Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1776,"":"io task","client":"0","":"downstairs","session_id":"f09f36ba-0c9d-4576-bf14-4efeee7f01ec"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:07.651673399Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1776,"":"io task","client":"1","":"downstairs","session_id":"f09f36ba-0c9d-4576-bf14-4efeee7f01ec"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:07.65179905Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1776,"task":"crutest"} {"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:07.651813468Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1776} {"msg":"cf9ae119-7675-4d5d-8b6e-1488d28d61aa active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:07.651837584Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1776,"session_id":"f09f36ba-0c9d-4576-bf14-4efeee7f01ec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:07.651847143Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1776,"client":"0","":"downstairs","session_id":"f09f36ba-0c9d-4576-bf14-4efeee7f01ec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:07.651856872Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1776,"client":"1","":"downstairs","session_id":"f09f36ba-0c9d-4576-bf14-4efeee7f01ec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:07.65186672Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1776,"client":"2","":"downstairs","session_id":"f09f36ba-0c9d-4576-bf14-4efeee7f01ec"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:07.651876349Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1776,"":"io task","client":"2","":"downstairs","session_id":"f09f36ba-0c9d-4576-bf14-4efeee7f01ec"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:07.651886467Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1776,"":"io task","client":"0","":"downstairs","session_id":"f09f36ba-0c9d-4576-bf14-4efeee7f01ec"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:07.651896546Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1776,"":"io task","client":"1","":"downstairs","session_id":"f09f36ba-0c9d-4576-bf14-4efeee7f01ec"} {"msg":"ds_connection connected from Ok(127.0.0.1:49964)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:07.651956917Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1776,"":"io task","client":"2","":"downstairs","session_id":"f09f36ba-0c9d-4576-bf14-4efeee7f01ec"} {"msg":"ds_connection connected from Ok(127.0.0.1:47890)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:07.651982623Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1776,"":"io task","client":"1","":"downstairs","session_id":"f09f36ba-0c9d-4576-bf14-4efeee7f01ec"} {"msg":"ds_connection connected from Ok(127.0.0.1:57389)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:07.651996111Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1776,"":"io task","client":"0","":"downstairs","session_id":"f09f36ba-0c9d-4576-bf14-4efeee7f01ec"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:07.652600752Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1776,"client":"2","":"downstairs","session_id":"f09f36ba-0c9d-4576-bf14-4efeee7f01ec"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:07.652618089Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1776,"client":"1","":"downstairs","session_id":"f09f36ba-0c9d-4576-bf14-4efeee7f01ec"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:07.652703407Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1776,"client":"0","":"downstairs","session_id":"f09f36ba-0c9d-4576-bf14-4efeee7f01ec"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:07.652923594Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1776,"session_id":"f09f36ba-0c9d-4576-bf14-4efeee7f01ec"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:07.652963198Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1776,"session_id":"f09f36ba-0c9d-4576-bf14-4efeee7f01ec"} {"msg":"[0]R flush_numbers[0..12]: [264, 271, 275, 275, 237, 276, 279, 276, 276, 276, 276, 280]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:07.652986855Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1776,"":"downstairs","":"downstairs","session_id":"f09f36ba-0c9d-4576-bf14-4efeee7f01ec"} {"msg":"[0]R generation[0..12]: [151, 154, 157, 157, 135, 158, 158, 158, 158, 158, 158, 159]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:07.652998873Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1776,"":"downstairs","":"downstairs","session_id":"f09f36ba-0c9d-4576-bf14-4efeee7f01ec"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:07.653010291Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1776,"":"downstairs","":"downstairs","session_id":"f09f36ba-0c9d-4576-bf14-4efeee7f01ec"} {"msg":"[1]R flush_numbers[0..12]: [264, 271, 275, 275, 237, 276, 279, 276, 276, 276, 276, 280]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:07.65302106Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1776,"":"downstairs","":"downstairs","session_id":"f09f36ba-0c9d-4576-bf14-4efeee7f01ec"} {"msg":"[1]R generation[0..12]: [151, 154, 157, 157, 135, 158, 158, 158, 158, 158, 158, 159]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:07.653032008Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1776,"":"downstairs","":"downstairs","session_id":"f09f36ba-0c9d-4576-bf14-4efeee7f01ec"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:07.653043086Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1776,"":"downstairs","":"downstairs","session_id":"f09f36ba-0c9d-4576-bf14-4efeee7f01ec"} {"msg":"[2]R flush_numbers[0..12]: [264, 271, 275, 275, 237, 276, 279, 276, 276, 276, 276, 280]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:07.653053865Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1776,"":"downstairs","":"downstairs","session_id":"f09f36ba-0c9d-4576-bf14-4efeee7f01ec"} {"msg":"[2]R generation[0..12]: [151, 154, 157, 157, 135, 158, 158, 158, 158, 158, 158, 159]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:07.653064583Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1776,"":"downstairs","":"downstairs","session_id":"f09f36ba-0c9d-4576-bf14-4efeee7f01ec"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:07.653077661Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1776,"":"downstairs","":"downstairs","session_id":"f09f36ba-0c9d-4576-bf14-4efeee7f01ec"} {"msg":"Max found gen is 160","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:07.65308717Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1776,"":"downstairs","":"downstairs","session_id":"f09f36ba-0c9d-4576-bf14-4efeee7f01ec"} {"msg":"Generation requested: 160 >= found:160","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:07.653097079Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1776,"":"downstairs","":"downstairs","session_id":"f09f36ba-0c9d-4576-bf14-4efeee7f01ec"} {"msg":"Next flush: 281","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:07.653106737Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1776,"":"downstairs","":"downstairs","session_id":"f09f36ba-0c9d-4576-bf14-4efeee7f01ec"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:07.653116386Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1776,"":"downstairs","":"downstairs","session_id":"f09f36ba-0c9d-4576-bf14-4efeee7f01ec"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:07.653126154Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1776,"session_id":"f09f36ba-0c9d-4576-bf14-4efeee7f01ec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:07.653134913Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1776,"client":"0","":"downstairs","session_id":"f09f36ba-0c9d-4576-bf14-4efeee7f01ec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:07.653144722Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1776,"client":"1","":"downstairs","session_id":"f09f36ba-0c9d-4576-bf14-4efeee7f01ec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:07.65315441Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1776,"client":"2","":"downstairs","session_id":"f09f36ba-0c9d-4576-bf14-4efeee7f01ec"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:07.653164179Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1776,"session_id":"f09f36ba-0c9d-4576-bf14-4efeee7f01ec"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:07.653173527Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1776,"session_id":"f09f36ba-0c9d-4576-bf14-4efeee7f01ec"} {"msg":"cf9ae119-7675-4d5d-8b6e-1488d28d61aa is now active with session: f09f36ba-0c9d-4576-bf14-4efeee7f01ec","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:07.653183506Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1776,"session_id":"f09f36ba-0c9d-4576-bf14-4efeee7f01ec"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:07.653192194Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1776,"session_id":"f09f36ba-0c9d-4576-bf14-4efeee7f01ec"} {"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:07.653201423Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1776} {"msg":"Activated sub_volume cf9ae119-7675-4d5d-8b6e-1488d28d61aa","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:07.653209352Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1776} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:07.653217531Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1776,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:07.65349423Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1776,"task":"crutest"} 01/30 Write block 132 len 16384 data: 8 11 15 14 02/30 Read block 334 len 4096 03/30 Write block 467 len 16384 data: 16 16 16 15 04/30 Read block 494 len 20480 05/30 Read block 453 len 4096 06/30 Write block 113 len 36864 data: 11 12 11 11 12 12 13 14 17 07/30 Write block 29 len 32768 data: 15 13 12 9 7 8 13 13 08/30 Read block 184 len 12288 09/30 Read block 407 len 32768 10/30 Write block 180 len 36864 data: 11 9 8 7 8 8 7 10 10 11/30 Read block 226 len 36864 12/30 Write block 12 len 24576 data: 7 8 7 13 13 13 13/30 Read block 177 len 4096 14/30 Read block 425 len 20480 15/30 Read block 94 len 8192 16/30 Write block 285 len 32768 data: 15 17 14 15 18 18 13 12 17/30 Write block 177 len 32768 data: 15 13 10 12 10 9 8 9 18/30 Flush 19/30 Write block 404 len 12288 data: 10 12 9 20/30 Write block 194 len 40960 data: 18 18 14 15 14 13 11 7 8 11 21/30 Read block 459 len 32768 22/30 Read block 129 len 32768 23/30 Read block 157 len 32768 24/30 Write block 56 len 40960 data: 15 11 10 7 10 9 10 9 9 8 25/30 Write block 209 len 32768 data: 16 13 10 12 12 12 12 9 26/30 Read block 571 len 16384 27/30 Read block 290 len 8192 28/30 Read block 508 len 36864 29/30 Write block 517 len 4096 data: 14 30/30 Write block 437 len 40960 data: 14 15 12 10 14 12 12 14 13 13 ---------------------------------------------------------------- Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Read 1 Sent Done Done false 1002 Acked Write 4 Sent Done Done false 1003 Acked Read 5 Sent Done Done false 1004 Acked Read 1 Sent Done Done false 1005 Acked Write 9 Sent Done Done false 1006 Acked Write 8 Sent Done Done false 1007 Acked Read 3 Sent Done Done false 1008 Acked Read 8 Sent Done Done false 1009 Acked Write 9 Sent Done Done false 1010 Acked Read 9 Sent Done Done false 1011 Acked Write 6 Sent Done Done false 1012 Acked Read 1 Sent Done Done false 1013 Acked Read 5 Sent Done Done false 1014 Acked Read 2 Sent Done Done false 1015 Acked Write 8 Sent Done Done false 1016 Acked Write 8 Sent Done Done false 1017 Acked Flush 0 Sent Done Done false 1018 Acked Write 3 Sent Done Done false 1019 Acked Write 10 Sent Done Done false 1020 Acked Read 8 Sent Done Done false 1021 Acked Read 8 Sent Done Done false 1022 Acked Read 8 Sent Done Done false 1023 Acked Write 10 Sent Done Done false 1024 Acked Write 8 Sent Done Done false 1025 Acked Read 4 Sent Done Done false 1026 Acked Read 2 Sent Done Done false 1027 Acked Read 9 Sent Done Done false 1028 Acked Write 1 Sent Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 1 1 31 Done 1 29 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:59:07.662769901Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1776} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:07.662948005Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1776,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 160 160 264 281 281 F F F <--- 1 020-039 154 160 160 271 281 281 F F F <--- 2 040-059 157 157 157 275 275 275 F T T <--- 3 060-079 157 157 157 275 275 275 F T T <--- 4 080-099 135 135 135 237 237 237 F F F 5 100-119 158 160 160 276 281 281 F F F <--- 6 120-139 158 160 160 279 281 281 T F F <--- 7 140-159 158 158 158 276 276 276 F F F 8 160-179 158 160 160 276 281 281 F F F <--- 9 180-199 158 160 160 276 281 281 F T T <--- 10 200-219 158 158 158 276 276 276 F T T <--- 11 220-239 159 159 159 280 280 280 F F F 12 240-259 151 151 151 264 264 264 F F F 13 260-279 152 152 152 267 267 267 F F F 14 280-299 158 160 160 277 281 281 F F F <--- 15 300-319 145 145 145 251 251 251 F F F 16 320-339 154 154 154 272 272 272 F F F 17 340-359 154 154 154 272 272 272 F F F 18 360-379 157 157 157 275 275 275 F F F 19 380-399 157 157 157 275 275 275 F F F 20 400-419 156 156 156 274 274 274 F T T <--- 21 420-439 157 157 157 275 275 275 F T T <--- 22 440-459 158 158 158 276 276 276 F T T <--- 23 460-479 157 160 160 275 281 281 F F F <--- 24 480-499 157 157 157 275 275 275 F F F 25 500-519 159 159 159 280 280 280 F T T <--- 26 520-539 155 155 155 273 273 273 F F F 27 540-559 158 158 158 278 278 278 F F F 28 560-579 158 158 158 277 277 277 F F F 29 580-599 157 157 157 275 275 275 F F F Max gen: 160, Max flush: 281 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 80 Verifying data now No change after verify Loop: 80 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 160 160 160 281 281 281 F F F 1 020-039 160 160 160 281 281 281 F F F 2 040-059 161 161 161 282 282 282 F F F 3 060-079 161 161 161 282 282 282 F F F 4 080-099 135 135 135 237 237 237 F F F 5 100-119 160 160 160 281 281 281 F F F 6 120-139 160 160 160 281 281 281 F F F 7 140-159 158 158 158 276 276 276 F F F 8 160-179 160 160 160 281 281 281 F F F 9 180-199 161 161 161 282 282 282 F F F 10 200-219 161 161 161 282 282 282 F F F 11 220-239 159 159 159 280 280 280 F F F 12 240-259 151 151 151 264 264 264 F F F 13 260-279 152 152 152 267 267 267 F F F 14 280-299 160 160 160 281 281 281 F F F 15 300-319 145 145 145 251 251 251 F F F 16 320-339 154 154 154 272 272 272 F F F 17 340-359 154 154 154 272 272 272 F F F 18 360-379 157 157 157 275 275 275 F F F 19 380-399 157 157 157 275 275 275 F F F 20 400-419 161 161 161 282 282 282 F F F 21 420-439 161 161 161 282 282 282 F F F 22 440-459 161 161 161 282 282 282 F F F 23 460-479 160 160 160 281 281 281 F F F 24 480-499 157 157 157 275 275 275 F F F 25 500-519 161 161 161 282 282 282 F F F 26 520-539 155 155 155 273 273 273 F F F 27 540-559 158 158 158 278 278 278 F F F 28 560-579 158 158 158 277 277 277 F F F 29 580-599 157 157 157 275 275 275 F F F Max gen: 161, Max flush: 282 Begin loop 81 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:11.920507813Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1784,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:11.928895715Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1784,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:11.92920345Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1784} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:11.929229386Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1784} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:11.929238245Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1784} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:11.929249773Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1784} {"msg":"Crucible ac0d5a92-2c94-4926-a6e8-26224f51b56c has session id: 29c9d3c0-d87b-40c5-9766-3a5706b272bd","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:11.929633146Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1784,"session_id":"29c9d3c0-d87b-40c5-9766-3a5706b272bd"} {"msg":"Upstairs opts: Upstairs UUID: ac0d5a92-2c94-4926-a6e8-26224f51b56c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:11.929667541Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1784,"session_id":"29c9d3c0-d87b-40c5-9766-3a5706b272bd"} {"msg":"Crucible stats registered with UUID: ac0d5a92-2c94-4926-a6e8-26224f51b56c","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:11.92967835Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1784,"session_id":"29c9d3c0-d87b-40c5-9766-3a5706b272bd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:59:11.929687858Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1784,"session_id":"29c9d3c0-d87b-40c5-9766-3a5706b272bd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:11.929837496Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1784,"":"io task","client":"2","":"downstairs","session_id":"29c9d3c0-d87b-40c5-9766-3a5706b272bd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:11.929869832Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1784,"":"io task","client":"0","":"downstairs","session_id":"29c9d3c0-d87b-40c5-9766-3a5706b272bd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:11.92988294Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1784,"":"io task","client":"1","":"downstairs","session_id":"29c9d3c0-d87b-40c5-9766-3a5706b272bd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:11.930010881Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1784,"task":"crutest"} {"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:11.930026898Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1784} {"msg":"ac0d5a92-2c94-4926-a6e8-26224f51b56c active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:11.930063863Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1784,"session_id":"29c9d3c0-d87b-40c5-9766-3a5706b272bd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:11.930073561Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1784,"client":"0","":"downstairs","session_id":"29c9d3c0-d87b-40c5-9766-3a5706b272bd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:11.93008342Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1784,"client":"1","":"downstairs","session_id":"29c9d3c0-d87b-40c5-9766-3a5706b272bd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:11.930092969Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1784,"client":"2","":"downstairs","session_id":"29c9d3c0-d87b-40c5-9766-3a5706b272bd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:11.930102567Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1784,"":"io task","client":"2","":"downstairs","session_id":"29c9d3c0-d87b-40c5-9766-3a5706b272bd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:11.930112766Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1784,"":"io task","client":"0","":"downstairs","session_id":"29c9d3c0-d87b-40c5-9766-3a5706b272bd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:11.930122784Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1784,"":"io task","client":"1","":"downstairs","session_id":"29c9d3c0-d87b-40c5-9766-3a5706b272bd"} {"msg":"ds_connection connected from Ok(127.0.0.1:58920)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:11.930184245Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1784,"":"io task","client":"0","":"downstairs","session_id":"29c9d3c0-d87b-40c5-9766-3a5706b272bd"} {"msg":"ds_connection connected from Ok(127.0.0.1:64571)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:11.930210681Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1784,"":"io task","client":"2","":"downstairs","session_id":"29c9d3c0-d87b-40c5-9766-3a5706b272bd"} {"msg":"ds_connection connected from Ok(127.0.0.1:64128)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:11.93022225Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1784,"":"io task","client":"1","":"downstairs","session_id":"29c9d3c0-d87b-40c5-9766-3a5706b272bd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:11.930819021Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1784,"client":"0","":"downstairs","session_id":"29c9d3c0-d87b-40c5-9766-3a5706b272bd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:11.930886192Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1784,"client":"2","":"downstairs","session_id":"29c9d3c0-d87b-40c5-9766-3a5706b272bd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:11.930905679Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1784,"client":"1","":"downstairs","session_id":"29c9d3c0-d87b-40c5-9766-3a5706b272bd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:11.931146723Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1784,"session_id":"29c9d3c0-d87b-40c5-9766-3a5706b272bd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:11.931195626Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1784,"session_id":"29c9d3c0-d87b-40c5-9766-3a5706b272bd"} {"msg":"[0]R flush_numbers[0..12]: [281, 281, 282, 282, 237, 281, 281, 276, 281, 282, 282, 280]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:11.931250158Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1784,"":"downstairs","":"downstairs","session_id":"29c9d3c0-d87b-40c5-9766-3a5706b272bd"} {"msg":"[0]R generation[0..12]: [160, 160, 161, 161, 135, 160, 160, 158, 160, 161, 161, 159]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:11.931261576Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1784,"":"downstairs","":"downstairs","session_id":"29c9d3c0-d87b-40c5-9766-3a5706b272bd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:11.931272745Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1784,"":"downstairs","":"downstairs","session_id":"29c9d3c0-d87b-40c5-9766-3a5706b272bd"} {"msg":"[1]R flush_numbers[0..12]: [281, 281, 282, 282, 237, 281, 281, 276, 281, 282, 282, 280]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:11.931283383Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1784,"":"downstairs","":"downstairs","session_id":"29c9d3c0-d87b-40c5-9766-3a5706b272bd"} {"msg":"[1]R generation[0..12]: [160, 160, 161, 161, 135, 160, 160, 158, 160, 161, 161, 159]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:11.931294051Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1784,"":"downstairs","":"downstairs","session_id":"29c9d3c0-d87b-40c5-9766-3a5706b272bd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:11.93130527Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1784,"":"downstairs","":"downstairs","session_id":"29c9d3c0-d87b-40c5-9766-3a5706b272bd"} {"msg":"[2]R flush_numbers[0..12]: [281, 281, 282, 282, 237, 281, 281, 276, 281, 282, 282, 280]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:11.931315988Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1784,"":"downstairs","":"downstairs","session_id":"29c9d3c0-d87b-40c5-9766-3a5706b272bd"} {"msg":"[2]R generation[0..12]: [160, 160, 161, 161, 135, 160, 160, 158, 160, 161, 161, 159]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:11.931326687Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1784,"":"downstairs","":"downstairs","session_id":"29c9d3c0-d87b-40c5-9766-3a5706b272bd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:11.931337845Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1784,"":"downstairs","":"downstairs","session_id":"29c9d3c0-d87b-40c5-9766-3a5706b272bd"} {"msg":"Max found gen is 162","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:11.931347394Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1784,"":"downstairs","":"downstairs","session_id":"29c9d3c0-d87b-40c5-9766-3a5706b272bd"} {"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:11.931357282Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1784,"":"downstairs","":"downstairs","session_id":"29c9d3c0-d87b-40c5-9766-3a5706b272bd"} {"msg":"Next flush: 283","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:11.931366781Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1784,"":"downstairs","":"downstairs","session_id":"29c9d3c0-d87b-40c5-9766-3a5706b272bd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:11.931377919Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1784,"":"downstairs","":"downstairs","session_id":"29c9d3c0-d87b-40c5-9766-3a5706b272bd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:11.931387778Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1784,"session_id":"29c9d3c0-d87b-40c5-9766-3a5706b272bd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:11.931396656Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1784,"client":"0","":"downstairs","session_id":"29c9d3c0-d87b-40c5-9766-3a5706b272bd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:11.931406435Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1784,"client":"1","":"downstairs","session_id":"29c9d3c0-d87b-40c5-9766-3a5706b272bd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:11.931415973Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1784,"client":"2","":"downstairs","session_id":"29c9d3c0-d87b-40c5-9766-3a5706b272bd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:11.931425892Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1784,"session_id":"29c9d3c0-d87b-40c5-9766-3a5706b272bd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:11.931435051Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1784,"session_id":"29c9d3c0-d87b-40c5-9766-3a5706b272bd"} {"msg":"ac0d5a92-2c94-4926-a6e8-26224f51b56c is now active with session: 29c9d3c0-d87b-40c5-9766-3a5706b272bd","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:11.931445149Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1784,"session_id":"29c9d3c0-d87b-40c5-9766-3a5706b272bd"} {"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:11.931454398Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1784} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:11.931461897Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1784,"session_id":"29c9d3c0-d87b-40c5-9766-3a5706b272bd"} {"msg":"Activated sub_volume ac0d5a92-2c94-4926-a6e8-26224f51b56c","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:11.931471075Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1784} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:11.931479244Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1784,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:11.931809775Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1784,"task":"crutest"} 01/30 Write block 139 len 8192 data: 13 14 02/30 Read block 445 len 40960 03/30 Write block 391 len 40960 data: 8 9 8 11 12 11 12 11 11 10 04/30 Read block 8 len 12288 05/30 Flush 06/30 Write block 195 len 12288 data: 19 15 16 07/30 Write block 235 len 28672 data: 12 11 13 12 11 11 10 08/30 Write block 227 len 36864 data: 9 11 13 14 12 12 10 8 13 09/30 Read block 75 len 36864 10/30 Write block 79 len 8192 data: 8 7 11/30 Flush 12/30 Write block 460 len 24576 data: 9 9 10 10 12 13 13/30 Flush 14/30 Write block 269 len 24576 data: 11 15 17 16 14 15 15/30 Write block 87 len 36864 data: 10 15 14 14 12 13 13 10 12 16/30 Flush 17/30 Write block 322 len 20480 data: 11 13 15 16 17 18/30 Read block 416 len 20480 19/30 Read block 427 len 24576 20/30 Read block 200 len 24576 21/30 Read block 180 len 24576 22/30 Read block 334 len 36864 23/30 Read block 139 len 32768 24/30 Write block 457 len 4096 data: 14 25/30 Write block 186 len 20480 data: 8 11 11 10 10 26/30 Write block 116 len 32768 data: 12 13 13 14 15 18 14 15 27/30 Write block 146 len 4096 data: 16 28/30 Read block 207 len 36864 29/30 Write block 53 len 12288 data: 11 14 13 30/30 Write block 549 len 24576 data: 14 13 14 17 14 15 ---------------------------------------------------------------- Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Read 10 Done Done Done false 1002 Acked Write 10 Done Sent Done false 1003 Acked Read 3 Done Sent Done false 1004 Acked Flush 0 Done Sent Done false 1005 Acked Write 3 Done Sent Done false 1006 Acked Write 7 Done Sent Done false 1007 Acked Write 9 Done Sent Done false 1008 Acked Read 9 Done Sent Done false 1009 Acked Write 2 Done Sent Done false 1010 Acked Flush 0 Done Sent Done false 1011 Acked Write 6 Done Sent Done false 1012 Acked Flush 0 Done Sent Done false 1013 Acked Write 6 Done Sent Done false 1014 Acked Write 9 Done Sent Done false 1015 Acked Flush 0 Done Sent Done false 1016 Acked Write 5 Done Sent Done false 1017 Acked Read 5 Done Sent Done false 1018 Acked Read 6 Done Sent Done false 1019 Acked Read 6 Done Sent Done false 1020 Acked Read 6 Done Sent Done false 1021 Acked Read 9 Done Sent Done false 1022 Acked Read 8 Done Sent Done false 1023 Acked Write 1 Done Sent Done false 1024 Acked Write 5 Done Sent Done false 1025 Acked Write 8 Done Sent Done false 1026 Acked Write 1 Done Sent Done false 1027 Acked Read 9 Done Sent Done false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 28 2 32 Done 28 2 28 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) None Some(JobId(1015)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:59:11.940230413Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1784} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:11.940404777Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1784,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 160 160 160 281 281 281 F F F 1 020-039 160 160 160 281 281 281 F F F 2 040-059 161 161 161 282 282 282 T F T <--- 3 060-079 162 161 162 284 282 284 F F F <--- 4 080-099 162 135 162 286 237 286 F F F <--- 5 100-119 160 160 160 281 281 281 T F T <--- 6 120-139 162 160 162 283 281 283 T T T <--- 7 140-159 162 158 162 283 276 283 T T T <--- 8 160-179 160 160 160 281 281 281 F F F 9 180-199 162 161 162 284 282 284 T F T <--- 10 200-219 161 161 161 282 282 282 F F F 11 220-239 162 159 162 284 280 284 F F F <--- 12 240-259 162 151 162 284 264 284 F F F <--- 13 260-279 162 152 162 286 267 286 F F F <--- 14 280-299 160 160 160 281 281 281 F F F 15 300-319 145 145 145 251 251 251 F F F 16 320-339 154 154 154 272 272 272 T F T <--- 17 340-359 154 154 154 272 272 272 F F F 18 360-379 157 157 157 275 275 275 F F F 19 380-399 162 157 162 283 275 283 F F F <--- 20 400-419 162 161 162 283 282 283 F F F <--- 21 420-439 161 161 161 282 282 282 F F F 22 440-459 161 161 161 282 282 282 T F T <--- 23 460-479 162 160 162 285 281 285 F F F <--- 24 480-499 157 157 157 275 275 275 F F F 25 500-519 161 161 161 282 282 282 F F F 26 520-539 155 155 155 273 273 273 F F F 27 540-559 158 158 158 278 278 278 T F T <--- 28 560-579 158 158 158 277 277 277 F F F 29 580-599 157 157 157 275 275 275 F F F Max gen: 162, Max flush: 286 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 81 Verifying data now No change after verify Loop: 81 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 160 160 160 281 281 281 F F F 1 020-039 160 160 160 281 281 281 F F F 2 040-059 163 163 163 287 287 287 F F F 3 060-079 162 162 162 284 284 284 F F F 4 080-099 162 162 162 286 286 286 F F F 5 100-119 163 163 163 287 287 287 F F F 6 120-139 163 163 163 287 287 287 F F F 7 140-159 163 163 163 287 287 287 F F F 8 160-179 160 160 160 281 281 281 F F F 9 180-199 163 163 163 287 287 287 F F F 10 200-219 161 161 161 282 282 282 F F F 11 220-239 162 162 162 284 284 284 F F F 12 240-259 162 162 162 284 284 284 F F F 13 260-279 162 162 162 286 286 286 F F F 14 280-299 160 160 160 281 281 281 F F F 15 300-319 145 145 145 251 251 251 F F F 16 320-339 163 163 163 287 287 287 F F F 17 340-359 154 154 154 272 272 272 F F F 18 360-379 157 157 157 275 275 275 F F F 19 380-399 162 162 162 283 283 283 F F F 20 400-419 162 162 162 283 283 283 F F F 21 420-439 161 161 161 282 282 282 F F F 22 440-459 163 163 163 287 287 287 F F F 23 460-479 162 162 162 285 285 285 F F F 24 480-499 157 157 157 275 275 275 F F F 25 500-519 161 161 161 282 282 282 F F F 26 520-539 155 155 155 273 273 273 F F F 27 540-559 163 163 163 287 287 287 F F F 28 560-579 158 158 158 277 277 277 F F F 29 580-599 157 157 157 275 275 275 F F F Max gen: 163, Max flush: 287 Begin loop 82 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:16.202438416Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1792,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:16.210491197Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1792,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:16.210792723Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1792} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:16.210826358Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1792} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:16.210838566Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1792} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:16.210850484Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1792} {"msg":"Crucible 24d261f7-544e-41f1-b2fd-0bcb85474318 has session id: f8b32d81-0f43-4c6e-a63d-5d0a04c882f0","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:16.211223339Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1792,"session_id":"f8b32d81-0f43-4c6e-a63d-5d0a04c882f0"} {"msg":"Upstairs opts: Upstairs UUID: 24d261f7-544e-41f1-b2fd-0bcb85474318, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:16.211248126Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1792,"session_id":"f8b32d81-0f43-4c6e-a63d-5d0a04c882f0"} {"msg":"Crucible stats registered with UUID: 24d261f7-544e-41f1-b2fd-0bcb85474318","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:16.211258734Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1792,"session_id":"f8b32d81-0f43-4c6e-a63d-5d0a04c882f0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:59:16.211268013Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1792,"session_id":"f8b32d81-0f43-4c6e-a63d-5d0a04c882f0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:16.21142022Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1792,"":"io task","client":"1","":"downstairs","session_id":"f8b32d81-0f43-4c6e-a63d-5d0a04c882f0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:16.211448866Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1792,"":"io task","client":"0","":"downstairs","session_id":"f8b32d81-0f43-4c6e-a63d-5d0a04c882f0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:16.211460384Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1792,"":"io task","client":"2","":"downstairs","session_id":"f8b32d81-0f43-4c6e-a63d-5d0a04c882f0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:16.211619981Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1792,"task":"crutest"} {"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:16.211631959Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1792} {"msg":"24d261f7-544e-41f1-b2fd-0bcb85474318 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:16.211645647Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1792,"session_id":"f8b32d81-0f43-4c6e-a63d-5d0a04c882f0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:16.211654576Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1792,"client":"0","":"downstairs","session_id":"f8b32d81-0f43-4c6e-a63d-5d0a04c882f0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:16.211664384Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1792,"client":"1","":"downstairs","session_id":"f8b32d81-0f43-4c6e-a63d-5d0a04c882f0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:16.211675993Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1792,"client":"2","":"downstairs","session_id":"f8b32d81-0f43-4c6e-a63d-5d0a04c882f0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:16.211685881Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1792,"":"io task","client":"2","":"downstairs","session_id":"f8b32d81-0f43-4c6e-a63d-5d0a04c882f0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:16.21169622Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1792,"":"io task","client":"0","":"downstairs","session_id":"f8b32d81-0f43-4c6e-a63d-5d0a04c882f0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:16.211706458Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1792,"":"io task","client":"1","":"downstairs","session_id":"f8b32d81-0f43-4c6e-a63d-5d0a04c882f0"} {"msg":"ds_connection connected from Ok(127.0.0.1:61377)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:16.211768149Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1792,"":"io task","client":"0","":"downstairs","session_id":"f8b32d81-0f43-4c6e-a63d-5d0a04c882f0"} {"msg":"ds_connection connected from Ok(127.0.0.1:37131)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:16.211789026Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1792,"":"io task","client":"1","":"downstairs","session_id":"f8b32d81-0f43-4c6e-a63d-5d0a04c882f0"} {"msg":"ds_connection connected from Ok(127.0.0.1:64837)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:16.211799924Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1792,"":"io task","client":"2","":"downstairs","session_id":"f8b32d81-0f43-4c6e-a63d-5d0a04c882f0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:16.21250633Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1792,"client":"0","":"downstairs","session_id":"f8b32d81-0f43-4c6e-a63d-5d0a04c882f0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:16.212539395Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1792,"client":"1","":"downstairs","session_id":"f8b32d81-0f43-4c6e-a63d-5d0a04c882f0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:16.212556683Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1792,"client":"2","":"downstairs","session_id":"f8b32d81-0f43-4c6e-a63d-5d0a04c882f0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:16.212817934Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1792,"session_id":"f8b32d81-0f43-4c6e-a63d-5d0a04c882f0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:16.21284939Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1792,"session_id":"f8b32d81-0f43-4c6e-a63d-5d0a04c882f0"} {"msg":"[0]R flush_numbers[0..12]: [281, 281, 287, 284, 286, 287, 287, 287, 281, 287, 282, 284]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:16.212924128Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1792,"":"downstairs","":"downstairs","session_id":"f8b32d81-0f43-4c6e-a63d-5d0a04c882f0"} {"msg":"[0]R generation[0..12]: [160, 160, 163, 162, 162, 163, 163, 163, 160, 163, 161, 162]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:16.212935587Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1792,"":"downstairs","":"downstairs","session_id":"f8b32d81-0f43-4c6e-a63d-5d0a04c882f0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:16.212946715Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1792,"":"downstairs","":"downstairs","session_id":"f8b32d81-0f43-4c6e-a63d-5d0a04c882f0"} {"msg":"[1]R flush_numbers[0..12]: [281, 281, 287, 284, 286, 287, 287, 287, 281, 287, 282, 284]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:16.212960093Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1792,"":"downstairs","":"downstairs","session_id":"f8b32d81-0f43-4c6e-a63d-5d0a04c882f0"} {"msg":"[1]R generation[0..12]: [160, 160, 163, 162, 162, 163, 163, 163, 160, 163, 161, 162]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:16.212970912Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1792,"":"downstairs","":"downstairs","session_id":"f8b32d81-0f43-4c6e-a63d-5d0a04c882f0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:16.21298182Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1792,"":"downstairs","":"downstairs","session_id":"f8b32d81-0f43-4c6e-a63d-5d0a04c882f0"} {"msg":"[2]R flush_numbers[0..12]: [281, 281, 287, 284, 286, 287, 287, 287, 281, 287, 282, 284]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:16.212992568Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1792,"":"downstairs","":"downstairs","session_id":"f8b32d81-0f43-4c6e-a63d-5d0a04c882f0"} {"msg":"[2]R generation[0..12]: [160, 160, 163, 162, 162, 163, 163, 163, 160, 163, 161, 162]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:16.213013475Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1792,"":"downstairs","":"downstairs","session_id":"f8b32d81-0f43-4c6e-a63d-5d0a04c882f0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:16.213024474Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1792,"":"downstairs","":"downstairs","session_id":"f8b32d81-0f43-4c6e-a63d-5d0a04c882f0"} {"msg":"Max found gen is 164","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:16.213034012Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1792,"":"downstairs","":"downstairs","session_id":"f8b32d81-0f43-4c6e-a63d-5d0a04c882f0"} {"msg":"Generation requested: 164 >= found:164","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:16.213043901Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1792,"":"downstairs","":"downstairs","session_id":"f8b32d81-0f43-4c6e-a63d-5d0a04c882f0"} {"msg":"Next flush: 288","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:16.213053419Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1792,"":"downstairs","":"downstairs","session_id":"f8b32d81-0f43-4c6e-a63d-5d0a04c882f0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:16.213062888Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1792,"":"downstairs","":"downstairs","session_id":"f8b32d81-0f43-4c6e-a63d-5d0a04c882f0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:16.213072917Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1792,"session_id":"f8b32d81-0f43-4c6e-a63d-5d0a04c882f0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:16.213081855Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1792,"client":"0","":"downstairs","session_id":"f8b32d81-0f43-4c6e-a63d-5d0a04c882f0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:16.213092064Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1792,"client":"1","":"downstairs","session_id":"f8b32d81-0f43-4c6e-a63d-5d0a04c882f0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:16.213101842Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1792,"client":"2","":"downstairs","session_id":"f8b32d81-0f43-4c6e-a63d-5d0a04c882f0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:16.213112081Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1792,"session_id":"f8b32d81-0f43-4c6e-a63d-5d0a04c882f0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:16.213122759Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1792,"session_id":"f8b32d81-0f43-4c6e-a63d-5d0a04c882f0"} {"msg":"24d261f7-544e-41f1-b2fd-0bcb85474318 is now active with session: f8b32d81-0f43-4c6e-a63d-5d0a04c882f0","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:16.213132948Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1792,"session_id":"f8b32d81-0f43-4c6e-a63d-5d0a04c882f0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:16.213141936Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1792,"session_id":"f8b32d81-0f43-4c6e-a63d-5d0a04c882f0"} {"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:16.213151135Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1792} {"msg":"Activated sub_volume 24d261f7-544e-41f1-b2fd-0bcb85474318","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:16.213159074Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1792} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:16.213167223Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1792,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:16.213480326Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1792,"task":"crutest"} 01/30 Write block 181 len 16384 data: 11 10 9 10 02/30 Write block 266 len 16384 data: 12 12 13 12 03/30 Write block 63 len 36864 data: 10 10 9 10 13 11 10 9 10 04/30 Write block 234 len 28672 data: 9 14 12 14 13 12 12 05/30 Read block 322 len 28672 06/30 Write block 57 len 4096 data: 12 07/30 Write block 353 len 4096 data: 9 08/30 Read block 245 len 36864 09/30 Write block 590 len 32768 data: 12 14 15 15 15 14 12 9 10/30 Read block 514 len 4096 11/30 Flush 12/30 Write block 308 len 8192 data: 11 14 13/30 Write block 330 len 40960 data: 14 13 11 10 9 10 10 12 13 14 14/30 Write block 447 len 20480 data: 14 16 15 16 18 15/30 Read block 160 len 8192 16/30 Write block 415 len 20480 data: 15 16 11 9 9 17/30 Write block 312 len 28672 data: 15 15 12 15 16 11 6 18/30 Write block 473 len 40960 data: 9 13 15 13 13 15 16 15 14 14 19/30 Write block 584 len 40960 data: 17 14 13 15 13 12 13 15 16 16 20/30 Write block 89 len 32768 data: 15 15 13 14 14 11 13 13 21/30 Write block 258 len 4096 data: 21 22/30 Write block 98 len 12288 data: 13 13 18 23/30 Read block 262 len 20480 24/30 Read block 431 len 36864 25/30 Flush 26/30 Flush 27/30 Write block 32 len 36864 data: 10 8 9 14 14 12 15 18 17 28/30 Read block 371 len 8192 29/30 Write block 30 len 32768 data: 14 13 11 9 10 15 15 13 30/30 Read block 527 len 16384 ---------------------------------------------------------------- Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Write 4 Done Done Done false 1002 Acked Write 9 Done Done Done false 1003 Acked Write 7 Done Done Done false 1004 Acked Read 7 Done Sent Done false 1005 Acked Write 1 Done Sent Done false 1006 Acked Write 1 Done Sent Done false 1007 Acked Read 9 Done Sent Done false 1008 Acked Write 8 Done Sent Done false 1009 Acked Read 1 Done Sent Done false 1010 Acked Flush 0 Done Sent Done false 1011 Acked Write 2 Done Sent Done false 1012 Acked Write 10 Done Sent Done false 1013 Acked Write 5 Done Sent Done false 1014 Acked Read 2 Done Sent Done false 1015 Acked Write 5 Done Sent Done false 1016 Acked Write 7 Done Sent Done false 1017 Acked Write 10 Done Sent Done false 1018 Acked Write 10 Done Sent Done false 1019 Acked Write 8 Done Sent Done false 1020 Acked Write 1 Done Sent Done false 1021 Acked Write 3 Done Sent Done false 1022 Acked Read 5 Done Sent Done false 1023 Acked Read 9 Done Sent Done false 1024 Acked Flush 0 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 9 Done Sent Done false 1027 Acked Read 2 Done Sent Done false 1028 Acked Write 8 Done Sent Done false 1029 Acked Read 4 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 26 0 27 Done 29 4 30 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) None Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:59:16.221772583Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1792} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:16.221961285Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1792,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 160 160 160 281 281 281 F F F 1 020-039 160 160 160 281 281 281 T F T <--- 2 040-059 164 163 164 288 287 288 T F T <--- 3 060-079 164 162 164 288 284 288 F T F <--- 4 080-099 164 162 164 289 286 289 F F F <--- 5 100-119 164 163 164 289 287 289 F F F <--- 6 120-139 163 163 163 287 287 287 F F F 7 140-159 163 163 163 287 287 287 F F F 8 160-179 160 160 160 281 281 281 F F F 9 180-199 164 163 164 288 287 288 F T F <--- 10 200-219 161 161 161 282 282 282 F F F 11 220-239 164 162 164 288 284 288 F T F <--- 12 240-259 164 162 164 289 284 289 F T F <--- 13 260-279 164 162 164 288 286 288 F T F <--- 14 280-299 160 160 160 281 281 281 F F F 15 300-319 164 145 164 289 251 289 F F F <--- 16 320-339 164 163 164 289 287 289 F F F <--- 17 340-359 164 154 164 288 272 288 F F F <--- 18 360-379 157 157 157 275 275 275 F F F 19 380-399 162 162 162 283 283 283 F F F 20 400-419 164 162 164 289 283 289 F F F <--- 21 420-439 161 161 161 282 282 282 F F F 22 440-459 164 163 164 289 287 289 F F F <--- 23 460-479 164 162 164 289 285 289 F F F <--- 24 480-499 164 157 164 289 275 289 F F F <--- 25 500-519 161 161 161 282 282 282 F F F 26 520-539 155 155 155 273 273 273 F F F 27 540-559 163 163 163 287 287 287 F F F 28 560-579 158 158 158 277 277 277 F F F 29 580-599 164 157 164 289 275 289 F F F <--- Max gen: 164, Max flush: 289 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 82 Verifying data now No change after verify Loop: 82 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 160 160 160 281 281 281 F F F 1 020-039 165 165 165 290 290 290 F F F 2 040-059 165 165 165 290 290 290 F F F 3 060-079 164 164 164 288 288 288 F F F 4 080-099 164 164 164 289 289 289 F F F 5 100-119 164 164 164 289 289 289 F F F 6 120-139 163 163 163 287 287 287 F F F 7 140-159 163 163 163 287 287 287 F F F 8 160-179 160 160 160 281 281 281 F F F 9 180-199 164 164 164 288 288 288 F F F 10 200-219 161 161 161 282 282 282 F F F 11 220-239 164 164 164 288 288 288 F F F 12 240-259 164 164 164 289 289 289 F F F 13 260-279 164 164 164 288 288 288 F F F 14 280-299 160 160 160 281 281 281 F F F 15 300-319 164 164 164 289 289 289 F F F 16 320-339 164 164 164 289 289 289 F F F 17 340-359 164 164 164 288 288 288 F F F 18 360-379 157 157 157 275 275 275 F F F 19 380-399 162 162 162 283 283 283 F F F 20 400-419 164 164 164 289 289 289 F F F 21 420-439 161 161 161 282 282 282 F F F 22 440-459 164 164 164 289 289 289 F F F 23 460-479 164 164 164 289 289 289 F F F 24 480-499 164 164 164 289 289 289 F F F 25 500-519 161 161 161 282 282 282 F F F 26 520-539 155 155 155 273 273 273 F F F 27 540-559 163 163 163 287 287 287 F F F 28 560-579 158 158 158 277 277 277 F F F 29 580-599 164 164 164 289 289 289 F F F Max gen: 165, Max flush: 290 Begin loop 83 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:20.480334643Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1800,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:20.488438378Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1800,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:20.488735344Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1800} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:20.4887605Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1800} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:20.488769789Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1800} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:20.488781537Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1800} {"msg":"Crucible 97e0c7ef-affb-4fe1-97ed-358ab0478169 has session id: 98b1e6d2-89db-4540-a2cd-ee898df4c6ce","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:20.489165491Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1800,"session_id":"98b1e6d2-89db-4540-a2cd-ee898df4c6ce"} {"msg":"Upstairs opts: Upstairs UUID: 97e0c7ef-affb-4fe1-97ed-358ab0478169, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:20.489186607Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1800,"session_id":"98b1e6d2-89db-4540-a2cd-ee898df4c6ce"} {"msg":"Crucible stats registered with UUID: 97e0c7ef-affb-4fe1-97ed-358ab0478169","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:20.489196606Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1800,"session_id":"98b1e6d2-89db-4540-a2cd-ee898df4c6ce"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:59:20.489205595Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1800,"session_id":"98b1e6d2-89db-4540-a2cd-ee898df4c6ce"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:20.489345134Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1800,"":"io task","client":"1","":"downstairs","session_id":"98b1e6d2-89db-4540-a2cd-ee898df4c6ce"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:20.489361452Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1800,"":"io task","client":"0","":"downstairs","session_id":"98b1e6d2-89db-4540-a2cd-ee898df4c6ce"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:20.48937133Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1800,"":"io task","client":"2","":"downstairs","session_id":"98b1e6d2-89db-4540-a2cd-ee898df4c6ce"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:20.489517249Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1800,"task":"crutest"} {"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:20.489532216Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1800} {"msg":"97e0c7ef-affb-4fe1-97ed-358ab0478169 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:20.489571681Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1800,"session_id":"98b1e6d2-89db-4540-a2cd-ee898df4c6ce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:20.489581079Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1800,"client":"0","":"downstairs","session_id":"98b1e6d2-89db-4540-a2cd-ee898df4c6ce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:20.489590838Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1800,"client":"1","":"downstairs","session_id":"98b1e6d2-89db-4540-a2cd-ee898df4c6ce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:20.489600566Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1800,"client":"2","":"downstairs","session_id":"98b1e6d2-89db-4540-a2cd-ee898df4c6ce"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:20.489610275Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1800,"":"io task","client":"2","":"downstairs","session_id":"98b1e6d2-89db-4540-a2cd-ee898df4c6ce"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:20.489620503Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1800,"":"io task","client":"0","":"downstairs","session_id":"98b1e6d2-89db-4540-a2cd-ee898df4c6ce"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:20.489630472Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1800,"":"io task","client":"1","":"downstairs","session_id":"98b1e6d2-89db-4540-a2cd-ee898df4c6ce"} {"msg":"ds_connection connected from Ok(127.0.0.1:40061)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:20.489720979Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1800,"":"io task","client":"0","":"downstairs","session_id":"98b1e6d2-89db-4540-a2cd-ee898df4c6ce"} {"msg":"ds_connection connected from Ok(127.0.0.1:50904)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:20.489749874Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1800,"":"io task","client":"1","":"downstairs","session_id":"98b1e6d2-89db-4540-a2cd-ee898df4c6ce"} {"msg":"ds_connection connected from Ok(127.0.0.1:37203)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:20.489761523Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1800,"":"io task","client":"2","":"downstairs","session_id":"98b1e6d2-89db-4540-a2cd-ee898df4c6ce"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:20.490503823Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1800,"client":"2","":"downstairs","session_id":"98b1e6d2-89db-4540-a2cd-ee898df4c6ce"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:20.49052619Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1800,"client":"0","":"downstairs","session_id":"98b1e6d2-89db-4540-a2cd-ee898df4c6ce"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:20.490537738Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1800,"client":"1","":"downstairs","session_id":"98b1e6d2-89db-4540-a2cd-ee898df4c6ce"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:20.490813737Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1800,"session_id":"98b1e6d2-89db-4540-a2cd-ee898df4c6ce"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:20.490851212Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1800,"session_id":"98b1e6d2-89db-4540-a2cd-ee898df4c6ce"} {"msg":"[0]R flush_numbers[0..12]: [281, 290, 290, 288, 289, 289, 287, 287, 281, 288, 282, 288]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:20.490904934Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1800,"":"downstairs","":"downstairs","session_id":"98b1e6d2-89db-4540-a2cd-ee898df4c6ce"} {"msg":"[0]R generation[0..12]: [160, 165, 165, 164, 164, 164, 163, 163, 160, 164, 161, 164]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:20.490916282Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1800,"":"downstairs","":"downstairs","session_id":"98b1e6d2-89db-4540-a2cd-ee898df4c6ce"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:20.490927221Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1800,"":"downstairs","":"downstairs","session_id":"98b1e6d2-89db-4540-a2cd-ee898df4c6ce"} {"msg":"[1]R flush_numbers[0..12]: [281, 290, 290, 288, 289, 289, 287, 287, 281, 288, 282, 288]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:20.490938049Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1800,"":"downstairs","":"downstairs","session_id":"98b1e6d2-89db-4540-a2cd-ee898df4c6ce"} {"msg":"[1]R generation[0..12]: [160, 165, 165, 164, 164, 164, 163, 163, 160, 164, 161, 164]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:20.490948447Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1800,"":"downstairs","":"downstairs","session_id":"98b1e6d2-89db-4540-a2cd-ee898df4c6ce"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:20.490959286Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1800,"":"downstairs","":"downstairs","session_id":"98b1e6d2-89db-4540-a2cd-ee898df4c6ce"} {"msg":"[2]R flush_numbers[0..12]: [281, 290, 290, 288, 289, 289, 287, 287, 281, 288, 282, 288]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:20.490969854Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1800,"":"downstairs","":"downstairs","session_id":"98b1e6d2-89db-4540-a2cd-ee898df4c6ce"} {"msg":"[2]R generation[0..12]: [160, 165, 165, 164, 164, 164, 163, 163, 160, 164, 161, 164]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:20.490982502Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1800,"":"downstairs","":"downstairs","session_id":"98b1e6d2-89db-4540-a2cd-ee898df4c6ce"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:20.490993581Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1800,"":"downstairs","":"downstairs","session_id":"98b1e6d2-89db-4540-a2cd-ee898df4c6ce"} {"msg":"Max found gen is 166","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:20.491003029Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1800,"":"downstairs","":"downstairs","session_id":"98b1e6d2-89db-4540-a2cd-ee898df4c6ce"} {"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:20.491012708Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1800,"":"downstairs","":"downstairs","session_id":"98b1e6d2-89db-4540-a2cd-ee898df4c6ce"} {"msg":"Next flush: 291","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:20.491022117Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1800,"":"downstairs","":"downstairs","session_id":"98b1e6d2-89db-4540-a2cd-ee898df4c6ce"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:20.491031575Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1800,"":"downstairs","":"downstairs","session_id":"98b1e6d2-89db-4540-a2cd-ee898df4c6ce"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:20.491041294Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1800,"session_id":"98b1e6d2-89db-4540-a2cd-ee898df4c6ce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:20.491050003Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1800,"client":"0","":"downstairs","session_id":"98b1e6d2-89db-4540-a2cd-ee898df4c6ce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:20.491059731Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1800,"client":"1","":"downstairs","session_id":"98b1e6d2-89db-4540-a2cd-ee898df4c6ce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:20.49106965Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1800,"client":"2","":"downstairs","session_id":"98b1e6d2-89db-4540-a2cd-ee898df4c6ce"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:20.491079608Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1800,"session_id":"98b1e6d2-89db-4540-a2cd-ee898df4c6ce"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:20.491088807Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1800,"session_id":"98b1e6d2-89db-4540-a2cd-ee898df4c6ce"} {"msg":"97e0c7ef-affb-4fe1-97ed-358ab0478169 is now active with session: 98b1e6d2-89db-4540-a2cd-ee898df4c6ce","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:20.491098725Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1800,"session_id":"98b1e6d2-89db-4540-a2cd-ee898df4c6ce"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:20.491107604Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1800,"session_id":"98b1e6d2-89db-4540-a2cd-ee898df4c6ce"} {"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:20.491116663Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1800} {"msg":"Activated sub_volume 97e0c7ef-affb-4fe1-97ed-358ab0478169","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:20.491124651Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1800} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:20.49113401Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1800,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:20.491428567Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1800,"task":"crutest"} 01/30 Write block 261 len 12288 data: 12 10 12 02/30 Write block 206 len 20480 data: 16 21 18 17 14 03/30 Write block 199 len 40960 data: 14 12 8 9 12 13 14 17 22 19 04/30 Read block 508 len 32768 05/30 Read block 121 len 36864 06/30 Read block 389 len 20480 07/30 Read block 170 len 36864 08/30 Read block 296 len 36864 09/30 Read block 357 len 12288 10/30 Write block 421 len 32768 data: 7 10 12 15 15 11 15 15 11/30 Write block 352 len 4096 data: 9 12/30 Read block 505 len 28672 13/30 Read block 77 len 12288 14/30 Flush 15/30 Write block 288 len 16384 data: 16 19 19 14 16/30 Write block 22 len 32768 data: 16 10 13 14 16 14 17 16 17/30 Write block 322 len 36864 data: 12 14 16 17 18 14 13 14 15 18/30 Read block 454 len 32768 19/30 Write block 173 len 24576 data: 12 12 13 15 16 14 20/30 Flush 21/30 Write block 547 len 32768 data: 11 11 15 14 15 18 15 16 22/30 Write block 496 len 16384 data: 10 9 8 11 23/30 Read block 496 len 12288 24/30 Write block 27 len 24576 data: 15 18 17 15 14 12 25/30 Read block 326 len 36864 26/30 Write block 9 len 36864 data: 8 7 6 8 9 8 14 14 14 27/30 Write block 221 len 36864 data: 11 12 11 13 10 9 10 12 14 28/30 Read block 532 len 32768 29/30 Read block 291 len 28672 30/30 Write block 448 len 36864 data: 17 16 17 19 17 18 15 10 11 ---------------------------------------------------------------- Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Sent Done false 1001 Acked Write 5 Done Sent Done false 1002 Acked Write 10 Done Sent Done false 1003 Acked Read 8 Done Sent Done false 1004 Acked Read 9 Done Sent Done false 1005 Acked Read 5 Done Sent Done false 1006 Acked Read 9 Done Sent Done false 1007 Acked Read 9 Done Sent Done false 1008 Acked Read 3 Done Sent Done false 1009 Acked Write 8 Done Sent Done false 1010 Acked Write 1 Done Sent Done false 1011 Acked Read 7 Done Sent Done false 1012 Acked Read 3 Done Sent Done false 1013 Acked Flush 0 Done Sent Done false 1014 Acked Write 4 Done Sent Done false 1015 Acked Write 8 Done Sent Done false 1016 Acked Write 9 Done Sent Done false 1017 Acked Read 8 Done Sent Done false 1018 Acked Write 6 Done Sent Done false 1019 Acked Flush 0 Done Sent Done false 1020 Acked Write 8 Done Sent Done false 1021 Acked Write 4 Done Sent Done false 1022 Acked Read 3 Done Sent Done false 1023 Acked Write 6 Done Sent Done false 1024 Acked Read 9 Done Sent Done false 1025 Acked Write 9 Done Sent Done false 1026 Acked Write 9 Done Sent Done false 1027 Acked Read 8 Done Sent Done false 1028 Acked Read 7 Done Sent Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 30 1 32 Done 29 0 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) None Some(JobId(1019)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:59:20.500805563Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1800} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:20.500993855Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1800,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 160 160 160 281 281 281 T F T <--- 1 020-039 166 165 166 292 290 292 T F T <--- 2 040-059 165 165 165 290 290 290 F F F 3 060-079 164 164 164 288 288 288 F F F 4 080-099 164 164 164 289 289 289 F F F 5 100-119 164 164 164 289 289 289 F F F 6 120-139 163 163 163 287 287 287 F F F 7 140-159 163 163 163 287 287 287 F F F 8 160-179 166 160 166 292 281 292 F F F <--- 9 180-199 166 164 166 291 288 291 F F F <--- 10 200-219 166 161 166 291 282 291 F F F <--- 11 220-239 164 164 164 288 288 288 T F T <--- 12 240-259 164 164 164 289 289 289 F F F 13 260-279 166 164 166 291 288 291 F F F <--- 14 280-299 166 160 166 292 281 292 F F F <--- 15 300-319 164 164 164 289 289 289 F F F 16 320-339 166 164 166 292 289 292 F F F <--- 17 340-359 166 164 166 291 288 291 F F F <--- 18 360-379 157 157 157 275 275 275 F F F 19 380-399 162 162 162 283 283 283 F F F 20 400-419 164 164 164 289 289 289 F F F 21 420-439 166 161 166 291 282 291 F F F <--- 22 440-459 164 164 164 289 289 289 T F T <--- 23 460-479 164 164 164 289 289 289 F F F 24 480-499 164 164 164 289 289 289 T F T <--- 25 500-519 161 161 161 282 282 282 F F F 26 520-539 155 155 155 273 273 273 F F F 27 540-559 163 163 163 287 287 287 T F T <--- 28 560-579 158 158 158 277 277 277 F F F 29 580-599 164 164 164 289 289 289 F F F Max gen: 166, Max flush: 292 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 83 Verifying data now No change after verify Loop: 83 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 293 293 293 F F F 1 020-039 167 167 167 293 293 293 F F F 2 040-059 165 165 165 290 290 290 F F F 3 060-079 164 164 164 288 288 288 F F F 4 080-099 164 164 164 289 289 289 F F F 5 100-119 164 164 164 289 289 289 F F F 6 120-139 163 163 163 287 287 287 F F F 7 140-159 163 163 163 287 287 287 F F F 8 160-179 166 166 166 292 292 292 F F F 9 180-199 166 166 166 291 291 291 F F F 10 200-219 166 166 166 291 291 291 F F F 11 220-239 167 167 167 293 293 293 F F F 12 240-259 164 164 164 289 289 289 F F F 13 260-279 166 166 166 291 291 291 F F F 14 280-299 166 166 166 292 292 292 F F F 15 300-319 164 164 164 289 289 289 F F F 16 320-339 166 166 166 292 292 292 F F F 17 340-359 166 166 166 291 291 291 F F F 18 360-379 157 157 157 275 275 275 F F F 19 380-399 162 162 162 283 283 283 F F F 20 400-419 164 164 164 289 289 289 F F F 21 420-439 166 166 166 291 291 291 F F F 22 440-459 167 167 167 293 293 293 F F F 23 460-479 164 164 164 289 289 289 F F F 24 480-499 167 167 167 293 293 293 F F F 25 500-519 161 161 161 282 282 282 F F F 26 520-539 155 155 155 273 273 273 F F F 27 540-559 167 167 167 293 293 293 F F F 28 560-579 158 158 158 277 277 277 F F F 29 580-599 164 164 164 289 289 289 F F F Max gen: 167, Max flush: 293 Begin loop 84 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:24.749885973Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1808,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:24.758209764Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1808,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:24.758502821Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1808} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:24.758527438Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1808} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:24.758535986Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1808} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:24.758547425Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1808} {"msg":"Crucible 91b58f18-858a-4c59-a0f1-edf72e63a70d has session id: 7f4b95d8-ec8c-4594-99b7-a0f48f95a490","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:24.758898333Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1808,"session_id":"7f4b95d8-ec8c-4594-99b7-a0f48f95a490"} {"msg":"Upstairs opts: Upstairs UUID: 91b58f18-858a-4c59-a0f1-edf72e63a70d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:24.75891961Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1808,"session_id":"7f4b95d8-ec8c-4594-99b7-a0f48f95a490"} {"msg":"Crucible stats registered with UUID: 91b58f18-858a-4c59-a0f1-edf72e63a70d","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:24.758929578Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1808,"session_id":"7f4b95d8-ec8c-4594-99b7-a0f48f95a490"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:59:24.758938907Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1808,"session_id":"7f4b95d8-ec8c-4594-99b7-a0f48f95a490"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:24.759082746Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1808,"":"io task","client":"0","":"downstairs","session_id":"7f4b95d8-ec8c-4594-99b7-a0f48f95a490"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:24.759113411Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1808,"":"io task","client":"2","":"downstairs","session_id":"7f4b95d8-ec8c-4594-99b7-a0f48f95a490"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:24.7591247Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1808,"":"io task","client":"1","":"downstairs","session_id":"7f4b95d8-ec8c-4594-99b7-a0f48f95a490"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:24.759264249Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1808,"task":"crutest"} {"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:24.759278427Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1808} {"msg":"91b58f18-858a-4c59-a0f1-edf72e63a70d active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:24.759316291Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1808,"session_id":"7f4b95d8-ec8c-4594-99b7-a0f48f95a490"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:24.75932565Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1808,"client":"0","":"downstairs","session_id":"7f4b95d8-ec8c-4594-99b7-a0f48f95a490"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:24.759335668Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1808,"client":"1","":"downstairs","session_id":"7f4b95d8-ec8c-4594-99b7-a0f48f95a490"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:24.759345597Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1808,"client":"2","":"downstairs","session_id":"7f4b95d8-ec8c-4594-99b7-a0f48f95a490"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:24.759355235Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1808,"":"io task","client":"2","":"downstairs","session_id":"7f4b95d8-ec8c-4594-99b7-a0f48f95a490"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:24.759365254Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1808,"":"io task","client":"0","":"downstairs","session_id":"7f4b95d8-ec8c-4594-99b7-a0f48f95a490"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:24.759375083Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1808,"":"io task","client":"1","":"downstairs","session_id":"7f4b95d8-ec8c-4594-99b7-a0f48f95a490"} {"msg":"ds_connection connected from Ok(127.0.0.1:55665)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:24.759418966Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1808,"":"io task","client":"2","":"downstairs","session_id":"7f4b95d8-ec8c-4594-99b7-a0f48f95a490"} {"msg":"ds_connection connected from Ok(127.0.0.1:42795)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:24.759444582Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1808,"":"io task","client":"1","":"downstairs","session_id":"7f4b95d8-ec8c-4594-99b7-a0f48f95a490"} {"msg":"ds_connection connected from Ok(127.0.0.1:48630)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:24.759518311Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1808,"":"io task","client":"0","":"downstairs","session_id":"7f4b95d8-ec8c-4594-99b7-a0f48f95a490"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:24.760124012Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1808,"client":"1","":"downstairs","session_id":"7f4b95d8-ec8c-4594-99b7-a0f48f95a490"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:24.760178404Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1808,"client":"0","":"downstairs","session_id":"7f4b95d8-ec8c-4594-99b7-a0f48f95a490"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:24.760192682Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1808,"client":"2","":"downstairs","session_id":"7f4b95d8-ec8c-4594-99b7-a0f48f95a490"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:24.760431227Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1808,"session_id":"7f4b95d8-ec8c-4594-99b7-a0f48f95a490"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:24.760449934Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1808,"session_id":"7f4b95d8-ec8c-4594-99b7-a0f48f95a490"} {"msg":"[0]R flush_numbers[0..12]: [293, 293, 290, 288, 289, 289, 287, 287, 292, 291, 291, 293]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:24.760496837Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1808,"":"downstairs","":"downstairs","session_id":"7f4b95d8-ec8c-4594-99b7-a0f48f95a490"} {"msg":"[0]R generation[0..12]: [167, 167, 165, 164, 164, 164, 163, 163, 166, 166, 166, 167]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:24.760508295Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1808,"":"downstairs","":"downstairs","session_id":"7f4b95d8-ec8c-4594-99b7-a0f48f95a490"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:24.760519414Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1808,"":"downstairs","":"downstairs","session_id":"7f4b95d8-ec8c-4594-99b7-a0f48f95a490"} {"msg":"[1]R flush_numbers[0..12]: [293, 293, 290, 288, 289, 289, 287, 287, 292, 291, 291, 293]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:24.760529812Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1808,"":"downstairs","":"downstairs","session_id":"7f4b95d8-ec8c-4594-99b7-a0f48f95a490"} {"msg":"[1]R generation[0..12]: [167, 167, 165, 164, 164, 164, 163, 163, 166, 166, 166, 167]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:24.760540431Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1808,"":"downstairs","":"downstairs","session_id":"7f4b95d8-ec8c-4594-99b7-a0f48f95a490"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:24.760551189Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1808,"":"downstairs","":"downstairs","session_id":"7f4b95d8-ec8c-4594-99b7-a0f48f95a490"} {"msg":"[2]R flush_numbers[0..12]: [293, 293, 290, 288, 289, 289, 287, 287, 292, 291, 291, 293]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:24.760561747Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1808,"":"downstairs","":"downstairs","session_id":"7f4b95d8-ec8c-4594-99b7-a0f48f95a490"} {"msg":"[2]R generation[0..12]: [167, 167, 165, 164, 164, 164, 163, 163, 166, 166, 166, 167]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:24.760572396Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1808,"":"downstairs","":"downstairs","session_id":"7f4b95d8-ec8c-4594-99b7-a0f48f95a490"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:24.760583264Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1808,"":"downstairs","":"downstairs","session_id":"7f4b95d8-ec8c-4594-99b7-a0f48f95a490"} {"msg":"Max found gen is 168","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:24.760592783Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1808,"":"downstairs","":"downstairs","session_id":"7f4b95d8-ec8c-4594-99b7-a0f48f95a490"} {"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:24.760602621Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1808,"":"downstairs","":"downstairs","session_id":"7f4b95d8-ec8c-4594-99b7-a0f48f95a490"} {"msg":"Next flush: 294","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:24.76061516Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1808,"":"downstairs","":"downstairs","session_id":"7f4b95d8-ec8c-4594-99b7-a0f48f95a490"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:24.760624948Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1808,"":"downstairs","":"downstairs","session_id":"7f4b95d8-ec8c-4594-99b7-a0f48f95a490"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:24.760634777Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1808,"session_id":"7f4b95d8-ec8c-4594-99b7-a0f48f95a490"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:24.760643535Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1808,"client":"0","":"downstairs","session_id":"7f4b95d8-ec8c-4594-99b7-a0f48f95a490"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:24.760653244Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1808,"client":"1","":"downstairs","session_id":"7f4b95d8-ec8c-4594-99b7-a0f48f95a490"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:24.760663143Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1808,"client":"2","":"downstairs","session_id":"7f4b95d8-ec8c-4594-99b7-a0f48f95a490"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:24.760672951Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1808,"session_id":"7f4b95d8-ec8c-4594-99b7-a0f48f95a490"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:24.76068206Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1808,"session_id":"7f4b95d8-ec8c-4594-99b7-a0f48f95a490"} {"msg":"91b58f18-858a-4c59-a0f1-edf72e63a70d is now active with session: 7f4b95d8-ec8c-4594-99b7-a0f48f95a490","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:24.760692148Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1808,"session_id":"7f4b95d8-ec8c-4594-99b7-a0f48f95a490"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:24.760700877Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1808,"session_id":"7f4b95d8-ec8c-4594-99b7-a0f48f95a490"} {"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:24.760709996Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1808} {"msg"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 :"Activated sub_volume 91b58f18-858a-4c59-a0f1-edf72e63a70d","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:24.760722784Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1808} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:24.760731013Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1808,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:24.760981426Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1808,"task":"crutest"} 01/30 Write block 563 len 36864 data: 9 10 9 6 6 6 7 9 9 02/30 Read block 438 len 32768 03/30 Read block 135 len 20480 04/30 Write block 586 len 8192 data: 14 16 05/30 Read block 150 len 36864 06/30 Read block 159 len 16384 07/30 Write block 477 len 8192 data: 14 16 08/30 Write block 454 len 32768 data: 16 11 12 15 10 10 10 10 09/30 Write block 128 len 24576 data: 8 10 8 9 9 12 10/30 Write block 477 len 24576 data: 15 17 17 16 15 15 11/30 Read block 581 len 40960 12/30 Read block 98 len 12288 13/30 Read block 125 len 4096 14/30 Read block 297 len 20480 15/30 Read block 588 len 12288 16/30 Write block 542 len 40960 data: 11 8 9 10 12 12 12 16 15 16 17/30 Write block 316 len 28672 data: 17 12 7 8 12 11 13 18/30 Write block 469 len 20480 data: 17 16 14 13 10 19/30 Write block 236 len 12288 data: 13 15 14 20/30 Write block 301 len 4096 data: 10 21/30 Read block 365 len 36864 22/30 Read block 197 len 28672 23/30 Read block 93 len 32768 24/30 Read block 515 len 24576 25/30 Read block 203 len 32768 26/30 Write block 259 len 24576 data: 18 11 13 11 13 12 27/30 Read block 461 len 16384 28/30 Write block 315 len 8192 data: 16 18 29/30 Write block 306 len 40960 data: 10 9 12 15 16 19 16 16 13 17 30/30 Write block 540 len 36864 data: 10 11 12 9 10 11 13 13 13 ---------------------------------------------------------------- Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Read 8 Done Done Sent false 1002 Acked Read 5 Done Done Sent false 1003 Acked Write 2 Done Done Sent false 1004 Acked Read 9 Done Done Sent false 1005 Acked Read 4 Done Done Sent false 1006 Acked Write 2 Done Done Sent false 1007 Acked Write 8 Done Done Sent false 1008 Acked Write 6 Done Done Sent false 1009 Acked Write 6 Done Done Sent false 1010 Acked Read 10 Done Done Sent false 1011 Acked Read 3 Done Done Sent false 1012 Acked Read 1 Done Done Sent false 1013 Acked Read 5 Done Done Sent false 1014 Acked Read 3 Done Done Sent false 1015 Acked Write 10 Done Done Sent false 1016 Acked Write 7 Done Done Sent false 1017 Acked Write 5 Done Done Sent false 1018 Acked Write 3 Done Done Sent false 1019 Acked Write 1 Done Done Sent false 1020 Acked Read 9 Done Done Sent false 1021 Acked Read 7 Done Done Sent false 1022 Acked Read 8 Done Done Sent false 1023 Acked Read 6 Done Done Sent false 1024 Acked Read 8 Done Done Sent false 1025 Acked Write 6 Done Done Sent false 1026 Acked Read 4 Done Done Sent false 1027 Acked Write 2 Done Done Sent false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 29 33 Done 28 28 1 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:59:24.769071852Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1808} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:24.769302178Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1808,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 293 293 293 F F F 1 020-039 167 167 167 293 293 293 F F F 2 040-059 165 165 165 290 290 290 F F F 3 060-079 164 164 164 288 288 288 F F F 4 080-099 164 164 164 289 289 289 F F F 5 100-119 164 164 164 289 289 289 F F F 6 120-139 163 163 163 287 287 287 T T F <--- 7 140-159 163 163 163 287 287 287 F F F 8 160-179 166 166 166 292 292 292 F F F 9 180-199 166 166 166 291 291 291 F F F 10 200-219 166 166 166 291 291 291 F F F 11 220-239 167 167 167 293 293 293 T T F <--- 12 240-259 164 164 164 289 289 289 T T F <--- 13 260-279 166 166 166 291 291 291 T T F <--- 14 280-299 166 166 166 292 292 292 F F F 15 300-319 164 164 164 289 289 289 T T F <--- 16 320-339 166 166 166 292 292 292 T T F <--- 17 340-359 166 166 166 291 291 291 F F F 18 360-379 157 157 157 275 275 275 F F F 19 380-399 162 162 162 283 283 283 F F F 20 400-419 164 164 164 289 289 289 F F F 21 420-439 166 166 166 291 291 291 F F F 22 440-459 167 167 167 293 293 293 T T F <--- 23 460-479 164 164 164 289 289 289 T T F <--- 24 480-499 167 167 167 293 293 293 T T F <--- 25 500-519 161 161 161 282 282 282 F F F 26 520-539 155 155 155 273 273 273 F F F 27 540-559 167 167 167 293 293 293 T T F <--- 28 560-579 158 158 158 277 277 277 T T T 29 580-599 164 164 164 289 289 289 T T F <--- Max gen: 167, Max flush: 293 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 84 Verifying data now No change after verify Loop: 84 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 293 293 293 F F F 1 020-039 167 167 167 293 293 293 F F F 2 040-059 165 165 165 290 290 290 F F F 3 060-079 164 164 164 288 288 288 F F F 4 080-099 164 164 164 289 289 289 F F F 5 100-119 164 164 164 289 289 289 F F F 6 120-139 168 168 168 294 294 294 F F F 7 140-159 163 163 163 287 287 287 F F F 8 160-179 166 166 166 292 292 292 F F F 9 180-199 166 166 166 291 291 291 F F F 10 200-219 166 166 166 291 291 291 F F F 11 220-239 168 168 168 294 294 294 F F F 12 240-259 168 168 168 294 294 294 F F F 13 260-279 168 168 168 294 294 294 F F F 14 280-299 166 166 166 292 292 292 F F F 15 300-319 168 168 168 294 294 294 F F F 16 320-339 168 168 168 294 294 294 F F F 17 340-359 166 166 166 291 291 291 F F F 18 360-379 157 157 157 275 275 275 F F F 19 380-399 162 162 162 283 283 283 F F F 20 400-419 164 164 164 289 289 289 F F F 21 420-439 166 166 166 291 291 291 F F F 22 440-459 168 168 168 294 294 294 F F F 23 460-479 168 168 168 294 294 294 F F F 24 480-499 168 168 168 294 294 294 F F F 25 500-519 161 161 161 282 282 282 F F F 26 520-539 155 155 155 273 273 273 F F F 27 540-559 168 168 168 294 294 294 F F F 28 560-579 168 168 168 294 294 294 F F F 29 580-599 168 168 168 294 294 294 F F F Max gen: 168, Max flush: 294 Begin loop 85 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:29.018614682Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1816,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:29.026709668Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1816,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:29.027000285Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1816} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:29.02703222Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1816} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:29.027041009Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1816} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:29.027052677Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1816} {"msg":"Crucible 48cc00bd-29fb-4d48-a661-892b505b3ce4 has session id: 61b62446-8474-4c82-95ea-f7deb441c92b","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:29.02737754Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1816,"session_id":"61b62446-8474-4c82-95ea-f7deb441c92b"} {"msg":"Upstairs opts: Upstairs UUID: 48cc00bd-29fb-4d48-a661-892b505b3ce4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:29.027397507Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1816,"session_id":"61b62446-8474-4c82-95ea-f7deb441c92b"} {"msg":"Crucible stats registered with UUID: 48cc00bd-29fb-4d48-a661-892b505b3ce4","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:29.027407765Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1816,"session_id":"61b62446-8474-4c82-95ea-f7deb441c92b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:59:29.027417064Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1816,"session_id":"61b62446-8474-4c82-95ea-f7deb441c92b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:29.027572181Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1816,"":"io task","client":"2","":"downstairs","session_id":"61b62446-8474-4c82-95ea-f7deb441c92b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:29.027603476Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1816,"":"io task","client":"0","":"downstairs","session_id":"61b62446-8474-4c82-95ea-f7deb441c92b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:29.027614605Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1816,"":"io task","client":"1","":"downstairs","session_id":"61b62446-8474-4c82-95ea-f7deb441c92b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:29.027721739Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1816,"task":"crutest"} {"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:29.027744865Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1816} {"msg":"48cc00bd-29fb-4d48-a661-892b505b3ce4 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:29.027759333Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1816,"session_id":"61b62446-8474-4c82-95ea-f7deb441c92b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:29.027768752Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1816,"client":"0","":"downstairs","session_id":"61b62446-8474-4c82-95ea-f7deb441c92b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:29.02778089Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1816,"client":"1","":"downstairs","session_id":"61b62446-8474-4c82-95ea-f7deb441c92b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:29.027790998Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1816,"client":"2","":"downstairs","session_id":"61b62446-8474-4c82-95ea-f7deb441c92b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:29.027800687Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1816,"":"io task","client":"2","":"downstairs","session_id":"61b62446-8474-4c82-95ea-f7deb441c92b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:29.027810776Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1816,"":"io task","client":"0","":"downstairs","session_id":"61b62446-8474-4c82-95ea-f7deb441c92b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:29.027820974Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1816,"":"io task","client":"1","":"downstairs","session_id":"61b62446-8474-4c82-95ea-f7deb441c92b"} {"msg":"ds_connection connected from Ok(127.0.0.1:57238)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:29.027883985Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1816,"":"io task","client":"2","":"downstairs","session_id":"61b62446-8474-4c82-95ea-f7deb441c92b"} {"msg":"ds_connection connected from Ok(127.0.0.1:35929)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:29.027900562Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1816,"":"io task","client":"1","":"downstairs","session_id":"61b62446-8474-4c82-95ea-f7deb441c92b"} {"msg":"ds_connection connected from Ok(127.0.0.1:51391)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:29.027940396Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1816,"":"io task","client":"0","":"downstairs","session_id":"61b62446-8474-4c82-95ea-f7deb441c92b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:29.028629165Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1816,"client":"2","":"downstairs","session_id":"61b62446-8474-4c82-95ea-f7deb441c92b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:29.028682437Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1816,"client":"1","":"downstairs","session_id":"61b62446-8474-4c82-95ea-f7deb441c92b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:29.028709483Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1816,"client":"0","":"downstairs","session_id":"61b62446-8474-4c82-95ea-f7deb441c92b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:29.028984562Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1816,"session_id":"61b62446-8474-4c82-95ea-f7deb441c92b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:29.029021357Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1816,"session_id":"61b62446-8474-4c82-95ea-f7deb441c92b"} {"msg":"[0]R flush_numbers[0..12]: [293, 293, 290, 288, 289, 289, 294, 287, 292, 291, 291, 294]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:29.029031825Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1816,"":"downstairs","":"downstairs","session_id":"61b62446-8474-4c82-95ea-f7deb441c92b"} {"msg":"[0]R generation[0..12]: [167, 167, 165, 164, 164, 164, 168, 163, 166, 166, 166, 168]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:29.029042834Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1816,"":"downstairs","":"downstairs","session_id":"61b62446-8474-4c82-95ea-f7deb441c92b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:29.029056772Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1816,"":"downstairs","":"downstairs","session_id":"61b62446-8474-4c82-95ea-f7deb441c92b"} {"msg":"[1]R flush_numbers[0..12]: [293, 293, 290, 288, 289, 289, 294, 287, 292, 291, 291, 294]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:29.02906754Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1816,"":"downstairs","":"downstairs","session_id":"61b62446-8474-4c82-95ea-f7deb441c92b"} {"msg":"[1]R generation[0..12]: [167, 167, 165, 164, 164, 164, 168, 163, 166, 166, 166, 168]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:29.029078439Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1816,"":"downstairs","":"downstairs","session_id":"61b62446-8474-4c82-95ea-f7deb441c92b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:29.029089497Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1816,"":"downstairs","":"downstairs","session_id":"61b62446-8474-4c82-95ea-f7deb441c92b"} {"msg":"[2]R flush_numbers[0..12]: [293, 293, 290, 288, 289, 289, 294, 287, 292, 291, 291, 294]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:29.029100365Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1816,"":"downstairs","":"downstairs","session_id":"61b62446-8474-4c82-95ea-f7deb441c92b"} {"msg":"[2]R generation[0..12]: [167, 167, 165, 164, 164, 164, 168, 163, 166, 166, 166, 168]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:29.029110914Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1816,"":"downstairs","":"downstairs","session_id":"61b62446-8474-4c82-95ea-f7deb441c92b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:29.029121792Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1816,"":"downstairs","":"downstairs","session_id":"61b62446-8474-4c82-95ea-f7deb441c92b"} {"msg":"Max found gen is 169","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:29.029131261Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1816,"":"downstairs","":"downstairs","session_id":"61b62446-8474-4c82-95ea-f7deb441c92b"} {"msg":"Generation requested: 170 >= found:169","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:29.029141419Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1816,"":"downstairs","":"downstairs","session_id":"61b62446-8474-4c82-95ea-f7deb441c92b"} {"msg":"Next flush: 295","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:29.029150938Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1816,"":"downstairs","":"downstairs","session_id":"61b62446-8474-4c82-95ea-f7deb441c92b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:29.029160566Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1816,"":"downstairs","":"downstairs","session_id":"61b62446-8474-4c82-95ea-f7deb441c92b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:29.029170195Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1816,"session_id":"61b62446-8474-4c82-95ea-f7deb441c92b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:29.029179054Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1816,"client":"0","":"downstairs","session_id":"61b62446-8474-4c82-95ea-f7deb441c92b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:29.029188852Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1816,"client":"1","":"downstairs","session_id":"61b62446-8474-4c82-95ea-f7deb441c92b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:29.029199811Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1816,"client":"2","":"downstairs","session_id":"61b62446-8474-4c82-95ea-f7deb441c92b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:29.029209659Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1816,"session_id":"61b62446-8474-4c82-95ea-f7deb441c92b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:29.029218948Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1816,"session_id":"61b62446-8474-4c82-95ea-f7deb441c92b"} {"msg":"48cc00bd-29fb-4d48-a661-892b505b3ce4 is now active with session: 61b62446-8474-4c82-95ea-f7deb441c92b","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:29.029228926Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1816,"session_id":"61b62446-8474-4c82-95ea-f7deb441c92b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:29.029237655Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1816,"session_id":"61b62446-8474-4c82-95ea-f7deb441c92b"} {"msg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:29.029246904Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1816} {"msg":"Activated sub_volume 48cc00bd-29fb-4d48-a661-892b505b3ce4","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:29.029254843Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1816} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:29.029263031Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1816,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:29.029557248Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1816,"task":"crutest"} 01/30 Write block 53 len 8192 data: 12 15 02/30 Write block 445 len 16384 data: 14 14 15 18 03/30 Read block 71 len 24576 04/30 Write block 568 len 24576 data: 7 8 10 10 9 10 05/30 Read block 130 len 28672 06/30 Write block 454 len 12288 data: 17 12 13 07/30 Read block 505 len 40960 08/30 Read block 176 len 40960 09/30 Write block 164 len 12288 data: 10 11 11 10/30 Read block 513 len 32768 11/30 Write block 543 len 40960 data: 10 11 12 14 14 14 17 16 17 19 12/30 Flush 13/30 Write block 252 len 24576 data: 13 15 18 16 18 19 14/30 Read block 305 len 24576 15/30 Write block 342 len 12288 data: 11 9 9 16/30 Write block 165 len 12288 data: 12 12 11 17/30 Read block 564 len 36864 18/30 Write block 54 len 12288 data: 16 14 16 19/30 Write block 569 len 16384 data: 9 11 11 10 20/30 Flush 21/30 Write block 455 len 20480 data: 13 14 16 11 11 22/30 Flush 23/30 Write block 513 len 40960 data: 17 17 16 15 15 16 14 13 13 10 24/30 Read block 143 len 36864 25/30 Read block 135 len 4096 26/30 Read block 68 len 32768 27/30 Read block 191 len 20480 28/30 Read block 242 len 4096 29/30 Write block 24 len 8192 data: 14 15 30/30 Write block 373 len 12288 data: 12 11 11 ---------------------------------------------------------------- Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Write 4 Done Done Sent false 1002 Acked Read 6 Done Done Sent false 1003 Acked Write 6 Done Done Sent false 1004 Acked Read 7 Done Done Sent false 1005 Acked Write 3 Done Done Sent false 1006 Acked Read 10 Done Done Sent false 1007 Acked Read 10 Done Done Sent false 1008 Acked Write 3 Done Done Sent false 1009 Acked Read 8 Done Done Sent false 1010 Acked Write 10 Done Done Sent false 1011 Acked Flush 0 Done Done Sent false 1012 Acked Write 6 Done Done Sent false 1013 Acked Read 6 Done Done Sent false 1014 Acked Write 3 Done Done Sent false 1015 Acked Write 3 Done Done Sent false 1016 Acked Read 9 Done Done Sent false 1017 Acked Write 3 Done Done Sent false 1018 Acked Write 4 Done Done Sent false 1019 Acked Flush 0 Done Done Sent false 1020 Acked Write 5 Done Done Sent false 1021 Acked Flush 0 Done Done Sent false 1022 Acked Write 10 Done Done Sent false 1023 Acked Read 9 Done Done Sent false 1024 Acked Read 1 Done Done Sent false 1025 Acked Read 8 Done Done Sent false 1026 Acked Read 5 Done Done Sent false 1027 Acked Read 1 Done Done Sent false 1028 Acked Write 2 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 29 33 Done 28 28 1 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:59:29.038109326Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1816} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:29.038279791Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1816,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 293 293 293 F F F 1 020-039 167 167 167 293 293 293 T T F <--- 2 040-059 170 170 165 296 296 290 F F T <--- 3 060-079 164 164 164 288 288 288 F F F 4 080-099 164 164 164 289 289 289 F F F 5 100-119 164 164 164 289 289 289 F F F 6 120-139 168 168 168 294 294 294 F F F 7 140-159 163 163 163 287 287 287 F F F 8 160-179 170 170 166 296 296 292 F F F <--- 9 180-199 166 166 166 291 291 291 F F F 10 200-219 166 166 166 291 291 291 F F F 11 220-239 168 168 168 294 294 294 F F F 12 240-259 170 170 168 296 296 294 F F F <--- 13 260-279 168 168 168 294 294 294 F F F 14 280-299 166 166 166 292 292 292 F F F 15 300-319 168 168 168 294 294 294 F F F 16 320-339 168 168 168 294 294 294 F F F 17 340-359 170 170 166 296 296 291 F F F <--- 18 360-379 157 157 157 275 275 275 T T F <--- 19 380-399 162 162 162 283 283 283 F F F 20 400-419 164 164 164 289 289 289 F F F 21 420-439 166 166 166 291 291 291 F F F 22 440-459 170 170 168 297 297 294 F F F <--- 23 460-479 168 168 168 294 294 294 F F F 24 480-499 168 168 168 294 294 294 F F F 25 500-519 161 161 161 282 282 282 T T F <--- 26 520-539 155 155 155 273 273 273 T T F <--- 27 540-559 170 170 168 295 295 294 F F F <--- 28 560-579 170 170 168 296 296 294 F F F <--- 29 580-599 168 168 168 294 294 294 F F F Max gen: 170, Max flush: 297 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 85 Verifying data now No change after verify Loop: 85 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 293 293 293 F F F 1 020-039 171 171 171 298 298 298 F F F 2 040-059 170 170 170 296 296 296 F F F 3 060-079 164 164 164 288 288 288 F F F 4 080-099 164 164 164 289 289 289 F F F 5 100-119 164 164 164 289 289 289 F F F 6 120-139 168 168 168 294 294 294 F F F 7 140-159 163 163 163 287 287 287 F F F 8 160-179 170 170 170 296 296 296 F F F 9 180-199 166 166 166 291 291 291 F F F 10 200-219 166 166 166 291 291 291 F F F 11 220-239 168 168 168 294 294 294 F F F 12 240-259 170 170 170 296 296 296 F F F 13 260-279 168 168 168 294 294 294 F F F 14 280-299 166 166 166 292 292 292 F F F 15 300-319 168 168 168 294 294 294 F F F 16 320-339 168 168 168 294 294 294 F F F 17 340-359 170 170 170 296 296 296 F F F 18 360-379 171 171 171 298 298 298 F F F 19 380-399 162 162 162 283 283 283 F F F 20 400-419 164 164 164 289 289 289 F F F 21 420-439 166 166 166 291 291 291 F F F 22 440-459 170 170 170 297 297 297 F F F 23 460-479 168 168 168 294 294 294 F F F 24 480-499 168 168 168 294 294 294 F F F 25 500-519 171 171 171 298 298 298 F F F 26 520-539 171 171 171 298 298 298 F F F 27 540-559 170 170 170 295 295 295 F F F 28 560-579 170 170 170 296 296 296 F F F 29 580-599 168 168 168 294 294 294 F F F Max gen: 171, Max flush: 298 Begin loop 86 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:33.283256446Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1824,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:33.291339183Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1824,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:33.29163436Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1824} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:33.291669865Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1824} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:33.291678393Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1824} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:33.291690092Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1824} {"msg":"Crucible 8b71ff44-1dce-4140-bf55-d685de1d5951 has session id: 4fa3007a-e0bb-40a4-9b8b-f3702aa1de14","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:33.292016703Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1824,"session_id":"4fa3007a-e0bb-40a4-9b8b-f3702aa1de14"} {"msg":"Upstairs opts: Upstairs UUID: 8b71ff44-1dce-4140-bf55-d685de1d5951, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:33.29203831Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1824,"session_id":"4fa3007a-e0bb-40a4-9b8b-f3702aa1de14"} {"msg":"Crucible stats registered with UUID: 8b71ff44-1dce-4140-bf55-d685de1d5951","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:33.292048379Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1824,"session_id":"4fa3007a-e0bb-40a4-9b8b-f3702aa1de14"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:59:33.292057427Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1824,"session_id":"4fa3007a-e0bb-40a4-9b8b-f3702aa1de14"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:33.292194517Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1824,"":"io task","client":"1","":"downstairs","session_id":"4fa3007a-e0bb-40a4-9b8b-f3702aa1de14"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:33.292225793Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1824,"":"io task","client":"0","":"downstairs","session_id":"4fa3007a-e0bb-40a4-9b8b-f3702aa1de14"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:33.292237201Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1824,"":"io task","client":"2","":"downstairs","session_id":"4fa3007a-e0bb-40a4-9b8b-f3702aa1de14"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:33.2923757Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1824,"task":"crutest"} {"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:33.292394738Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1824} {"msg":"8b71ff44-1dce-4140-bf55-d685de1d5951 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:33.292425363Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1824,"session_id":"4fa3007a-e0bb-40a4-9b8b-f3702aa1de14"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:33.292434882Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1824,"client":"0","":"downstairs","session_id":"4fa3007a-e0bb-40a4-9b8b-f3702aa1de14"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:33.2924445Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1824,"client":"1","":"downstairs","session_id":"4fa3007a-e0bb-40a4-9b8b-f3702aa1de14"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:33.292454139Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1824,"client":"2","":"downstairs","session_id":"4fa3007a-e0bb-40a4-9b8b-f3702aa1de14"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:33.292463847Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1824,"":"io task","client":"2","":"downstairs","session_id":"4fa3007a-e0bb-40a4-9b8b-f3702aa1de14"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:33.292473926Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1824,"":"io task","client":"0","":"downstairs","session_id":"4fa3007a-e0bb-40a4-9b8b-f3702aa1de14"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:33.292483964Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1824,"":"io task","client":"1","":"downstairs","session_id":"4fa3007a-e0bb-40a4-9b8b-f3702aa1de14"} {"msg":"ds_connection connected from Ok(127.0.0.1:42324)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:33.292554554Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1824,"":"io task","client":"2","":"downstairs","session_id":"4fa3007a-e0bb-40a4-9b8b-f3702aa1de14"} {"msg":"ds_connection connected from Ok(127.0.0.1:47589)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:33.292565582Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1824,"":"io task","client":"1","":"downstairs","session_id":"4fa3007a-e0bb-40a4-9b8b-f3702aa1de14"} {"msg":"ds_connection connected from Ok(127.0.0.1:50431)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:33.292575661Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1824,"":"io task","client":"0","":"downstairs","session_id":"4fa3007a-e0bb-40a4-9b8b-f3702aa1de14"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:33.293276997Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1824,"client":"0","":"downstairs","session_id":"4fa3007a-e0bb-40a4-9b8b-f3702aa1de14"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:33.293302744Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1824,"client":"1","":"downstairs","session_id":"4fa3007a-e0bb-40a4-9b8b-f3702aa1de14"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:33.293314712Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1824,"client":"2","":"downstairs","session_id":"4fa3007a-e0bb-40a4-9b8b-f3702aa1de14"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:33.293593711Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1824,"session_id":"4fa3007a-e0bb-40a4-9b8b-f3702aa1de14"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:33.293603289Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1824,"session_id":"4fa3007a-e0bb-40a4-9b8b-f3702aa1de14"} {"msg":"[0]R flush_numbers[0..12]: [293, 298, 296, 288, 289, 289, 294, 287, 296, 291, 291, 294]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:33.293620647Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1824,"":"downstairs","":"downstairs","session_id":"4fa3007a-e0bb-40a4-9b8b-f3702aa1de14"} {"msg":"[0]R generation[0..12]: [167, 171, 170, 164, 164, 164, 168, 163, 170, 166, 166, 168]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:33.293631455Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1824,"":"downstairs","":"downstairs","session_id":"4fa3007a-e0bb-40a4-9b8b-f3702aa1de14"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:33.293642763Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1824,"":"downstairs","":"downstairs","session_id":"4fa3007a-e0bb-40a4-9b8b-f3702aa1de14"} {"msg":"[1]R flush_numbers[0..12]: [293, 298, 296, 288, 289, 289, 294, 287, 296, 291, 291, 294]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:33.293653412Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1824,"":"downstairs","":"downstairs","session_id":"4fa3007a-e0bb-40a4-9b8b-f3702aa1de14"} {"msg":"[1]R generation[0..12]: [167, 171, 170, 164, 164, 164, 168, 163, 170, 166, 166, 168]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:33.29366398Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1824,"":"downstairs","":"downstairs","session_id":"4fa3007a-e0bb-40a4-9b8b-f3702aa1de14"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:33.293675089Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1824,"":"downstairs","":"downstairs","session_id":"4fa3007a-e0bb-40a4-9b8b-f3702aa1de14"} {"msg":"[2]R flush_numbers[0..12]: [293, 298, 296, 288, 289, 289, 294, 287, 296, 291, 291, 294]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:33.293687977Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1824,"":"downstairs","":"downstairs","session_id":"4fa3007a-e0bb-40a4-9b8b-f3702aa1de14"} {"msg":"[2]R generation[0..12]: [167, 171, 170, 164, 164, 164, 168, 163, 170, 166, 166, 168]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:33.293698485Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1824,"":"downstairs","":"downstairs","session_id":"4fa3007a-e0bb-40a4-9b8b-f3702aa1de14"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:33.293709484Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1824,"":"downstairs","":"downstairs","session_id":"4fa3007a-e0bb-40a4-9b8b-f3702aa1de14"} {"msg":"Max found gen is 172","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:33.293718792Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1824,"":"downstairs","":"downstairs","session_id":"4fa3007a-e0bb-40a4-9b8b-f3702aa1de14"} {"msg":"Generation requested: 172 >= found:172","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:33.293728651Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1824,"":"downstairs","":"downstairs","session_id":"4fa3007a-e0bb-40a4-9b8b-f3702aa1de14"} {"msg":"Next flush: 299","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:33.293737959Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1824,"":"downstairs","":"downstairs","session_id":"4fa3007a-e0bb-40a4-9b8b-f3702aa1de14"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:33.293747288Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1824,"":"downstairs","":"downstairs","session_id":"4fa3007a-e0bb-40a4-9b8b-f3702aa1de14"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:33.293757267Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1824,"session_id":"4fa3007a-e0bb-40a4-9b8b-f3702aa1de14"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:33.293766185Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1824,"client":"0","":"downstairs","session_id":"4fa3007a-e0bb-40a4-9b8b-f3702aa1de14"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:33.293776354Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1824,"client":"1","":"downstairs","session_id":"4fa3007a-e0bb-40a4-9b8b-f3702aa1de14"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:33.293786162Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1824,"client":"2","":"downstairs","session_id":"4fa3007a-e0bb-40a4-9b8b-f3702aa1de14"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:33.293796101Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1824,"session_id":"4fa3007a-e0bb-40a4-9b8b-f3702aa1de14"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:33.293805369Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1824,"session_id":"4fa3007a-e0bb-40a4-9b8b-f3702aa1de14"} {"msg":"8b71ff44-1dce-4140-bf55-d685de1d5951 is now active with session: 4fa3007a-e0bb-40a4-9b8b-f3702aa1de14","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:33.293815158Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1824,"session_id":"4fa3007a-e0bb-40a4-9b8b-f3702aa1de14"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:33.293823927Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1824,"session_id":"4fa3007a-e0bb-40a4-9b8b-f3702aa1de14"} {"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:33.293832795Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1824} {"msg":"Activated sub_volume 8b71ff44-1dce-4140-bf55-d685de1d5951","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:33.293842144Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1824} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:33.293850013Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1824,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:33.29414372Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1824,"task":"crutest"} 01/30 Write block 480 len 24576 data: 17 16 16 12 10 12 02/30 Write block 191 len 16384 data: 17 19 19 19 03/30 Flush 04/30 Write block 24 len 12288 data: 15 16 17 05/30 Write block 537 len 16384 data: 11 10 9 11 06/30 Write block 259 len 32768 data: 19 12 14 12 14 13 10 13 07/30 Read block 164 len 16384 08/30 Write block 293 len 8192 data: 11 8 09/30 Read block 95 len 32768 10/30 Read block 56 len 8192 11/30 Read block 316 len 36864 12/30 Read block 94 len 36864 13/30 Write block 183 len 8192 data: 10 11 14/30 Write block 143 len 4096 data: 15 15/30 Write block 457 len 12288 data: 17 12 12 16/30 Read block 93 len 40960 17/30 Read block 212 len 40960 18/30 Write block 221 len 8192 data: 12 13 19/30 Read block 20 len 8192 20/30 Read block 453 len 4096 21/30 Write block 482 len 8192 data: 17 13 22/30 Read block 84 len 24576 23/30 Write block 340 len 28672 data: 13 12 12 10 10 8 8 24/30 Flush 25/30 Flush 26/30 Write block 373 len 40960 data: 13 12 12 11 11 8 10 10 12 16 27/30 Write block 445 len 24576 data: 15 15 16 19 17 18 28/30 Write block 115 len 12288 data: 12 13 14 29/30 Read block 426 len 12288 30/30 Read block 555 len 24576 ---------------------------------------------------------------- Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Write 4 Done Sent Done false 1002 Acked Flush 0 Done Sent Done false 1003 Acked Write 3 Done Sent Done false 1004 Acked Write 4 Done Sent Done false 1005 Acked Write 8 Done Sent Done false 1006 Acked Read 4 Done Sent Done false 1007 Acked Write 2 Done Sent Done false 1008 Acked Read 8 Done Sent Done false 1009 Acked Read 2 Done Sent Done false 1010 Acked Read 9 Done Sent Done false 1011 Acked Read 9 Done Sent Done false 1012 Acked Write 2 Done Sent Done false 1013 Acked Write 1 Done Sent Done false 1014 Acked Write 3 Done Sent Done false 1015 Acked Read 10 Done Sent Done false 1016 Acked Read 10 Done Sent Done false 1017 Acked Write 2 Done Sent Done false 1018 Acked Read 2 Done Sent Done false 1019 Acked Read 1 Done Sent Done false 1020 Acked Write 2 Done Sent Done false 1021 Acked Read 6 Done Sent Done false 1022 Acked Write 7 Done Sent Done false 1023 Acked Flush 0 Done Sent Done false 1024 Acked Flush 0 Done Sent Done false 1025 Acked Write 10 Done Sent Done false 1026 Acked Write 6 Done Sent Done false 1027 Acked Write 3 Done Sent Done false 1028 Acked Read 3 Done Sent Done false 1029 Acked Read 6 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 29 0 29 Done 30 1 30 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) None Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:59:33.301708893Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1824} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:33.301889447Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1824,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 293 293 293 F F F 1 020-039 172 171 172 300 298 300 F F F <--- 2 040-059 170 170 170 296 296 296 F F F 3 060-079 164 164 164 288 288 288 F F F 4 080-099 164 164 164 289 289 289 F F F 5 100-119 164 164 164 289 289 289 T F T <--- 6 120-139 168 168 168 294 294 294 F F F 7 140-159 172 163 172 300 287 300 F F F <--- 8 160-179 170 170 170 296 296 296 F F F 9 180-199 172 166 172 300 291 300 F F F <--- 10 200-219 166 166 166 291 291 291 F F F 11 220-239 172 168 172 300 294 300 F F F <--- 12 240-259 172 170 172 300 296 300 F F F <--- 13 260-279 172 168 172 300 294 300 F F F <--- 14 280-299 172 166 172 300 292 300 F F F <--- 15 300-319 168 168 168 294 294 294 F F F 16 320-339 168 168 168 294 294 294 F F F 17 340-359 172 170 172 300 296 300 F F F <--- 18 360-379 171 171 171 298 298 298 T F T <--- 19 380-399 162 162 162 283 283 283 T F T <--- 20 400-419 164 164 164 289 289 289 F F F 21 420-439 166 166 166 291 291 291 F F F 22 440-459 172 170 172 300 297 300 T F T <--- 23 460-479 168 168 168 294 294 294 F F F 24 480-499 172 168 172 300 294 300 F T F <--- 25 500-519 171 171 171 298 298 298 F F F 26 520-539 172 171 172 300 298 300 F F F <--- 27 540-559 172 170 172 300 295 300 F F F <--- 28 560-579 170 170 170 296 296 296 F F F 29 580-599 168 168 168 294 294 294 F F F Max gen: 172, Max flush: 300 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 86 Verifying data now No change after verify Loop: 86 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 293 293 293 F F F 1 020-039 172 172 172 300 300 300 F F F 2 040-059 170 170 170 296 296 296 F F F 3 060-079 164 164 164 288 288 288 F F F 4 080-099 164 164 164 289 289 289 F F F 5 100-119 173 173 173 301 301 301 F F F 6 120-139 168 168 168 294 294 294 F F F 7 140-159 172 172 172 300 300 300 F F F 8 160-179 170 170 170 296 296 296 F F F 9 180-199 172 172 172 300 300 300 F F F 10 200-219 166 166 166 291 291 291 F F F 11 220-239 172 172 172 300 300 300 F F F 12 240-259 172 172 172 300 300 300 F F F 13 260-279 172 172 172 300 300 300 F F F 14 280-299 172 172 172 300 300 300 F F F 15 300-319 168 168 168 294 294 294 F F F 16 320-339 168 168 168 294 294 294 F F F 17 340-359 172 172 172 300 300 300 F F F 18 360-379 173 173 173 301 301 301 F F F 19 380-399 173 173 173 301 301 301 F F F 20 400-419 164 164 164 289 289 289 F F F 21 420-439 166 166 166 291 291 291 F F F 22 440-459 173 173 173 301 301 301 F F F 23 460-479 168 168 168 294 294 294 F F F 24 480-499 172 172 172 300 300 300 F F F 25 500-519 171 171 171 298 298 298 F F F 26 520-539 172 172 172 300 300 300 F F F 27 540-559 172 172 172 300 300 300 F F F 28 560-579 170 170 170 296 296 296 F F F 29 580-599 168 168 168 294 294 294 F F F Max gen: 173, Max flush: 301 Begin loop 87 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:37.555463873Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1832,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:59:37.562558876Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1832,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:37.562614508Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1832,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:37.569457119Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1832,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:37.56972155Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1832} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:37.569745856Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1832} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:37.569754435Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1832} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:37.569766163Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1832} {"msg":"Crucible e1aae5dc-de37-42c8-8b54-5b8174202406 has session id: 2a65786e-3920-4163-a002-f88b981c014a","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:37.570086186Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1832,"session_id":"2a65786e-3920-4163-a002-f88b981c014a"} {"msg":"Upstairs opts: Upstairs UUID: e1aae5dc-de37-42c8-8b54-5b8174202406, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:37.570117061Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1832,"session_id":"2a65786e-3920-4163-a002-f88b981c014a"} {"msg":"Crucible stats registered with UUID: e1aae5dc-de37-42c8-8b54-5b8174202406","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:37.57012835Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1832,"session_id":"2a65786e-3920-4163-a002-f88b981c014a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:59:37.570140218Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1832,"session_id":"2a65786e-3920-4163-a002-f88b981c014a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:37.570276898Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1832,"":"io task","client":"2","":"downstairs","session_id":"2a65786e-3920-4163-a002-f88b981c014a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:37.570309583Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1832,"":"io task","client":"0","":"downstairs","session_id":"2a65786e-3920-4163-a002-f88b981c014a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:37.570320671Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1832,"":"io task","client":"1","":"downstairs","session_id":"2a65786e-3920-4163-a002-f88b981c014a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:37.57046097Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1832,"task":"crutest"} {"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:37.570479158Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1832} {"msg":"e1aae5dc-de37-42c8-8b54-5b8174202406 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:37.570497435Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1832,"session_id":"2a65786e-3920-4163-a002-f88b981c014a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:37.570506444Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1832,"client":"0","":"downstairs","session_id":"2a65786e-3920-4163-a002-f88b981c014a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:37.570516132Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1832,"client":"1","":"downstairs","session_id":"2a65786e-3920-4163-a002-f88b981c014a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:37.570525841Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1832,"client":"2","":"downstairs","session_id":"2a65786e-3920-4163-a002-f88b981c014a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:37.57053531Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1832,"":"io task","client":"2","":"downstairs","session_id":"2a65786e-3920-4163-a002-f88b981c014a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:37.570545378Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1832,"":"io task","client":"0","":"downstairs","session_id":"2a65786e-3920-4163-a002-f88b981c014a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:37.570569434Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1832,"":"io task","client":"1","":"downstairs","session_id":"2a65786e-3920-4163-a002-f88b981c014a"} {"msg":"ds_connection connected from Ok(127.0.0.1:35829)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:37.570619227Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1832,"":"io task","client":"1","":"downstairs","session_id":"2a65786e-3920-4163-a002-f88b981c014a"} {"msg":"ds_connection connected from Ok(127.0.0.1:51481)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:37.570632055Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1832,"":"io task","client":"0","":"downstairs","session_id":"2a65786e-3920-4163-a002-f88b981c014a"} {"msg":"ds_connection connected from Ok(127.0.0.1:43747)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:37.570727281Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1832,"":"io task","client":"2","":"downstairs","session_id":"2a65786e-3920-4163-a002-f88b981c014a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:37.571354279Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1832,"client":"1","":"downstairs","session_id":"2a65786e-3920-4163-a002-f88b981c014a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:37.571393113Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1832,"client":"2","":"downstairs","session_id":"2a65786e-3920-4163-a002-f88b981c014a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:37.571458853Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1832,"client":"0","":"downstairs","session_id":"2a65786e-3920-4163-a002-f88b981c014a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:37.571650825Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1832,"session_id":"2a65786e-3920-4163-a002-f88b981c014a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:37.571668282Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1832,"session_id":"2a65786e-3920-4163-a002-f88b981c014a"} {"msg":"[0]R flush_numbers[0..12]: [293, 300, 296, 288, 289, 301, 294, 300, 296, 300, 291, 300]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:37.571731233Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1832,"":"downstairs","":"downstairs","session_id":"2a65786e-3920-4163-a002-f88b981c014a"} {"msg":"[0]R generation[0..12]: [167, 172, 170, 164, 164, 173, 168, 172, 170, 172, 166, 172]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:37.571742911Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1832,"":"downstairs","":"downstairs","session_id":"2a65786e-3920-4163-a002-f88b981c014a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:37.57175397Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1832,"":"downstairs","":"downstairs","session_id":"2a65786e-3920-4163-a002-f88b981c014a"} {"msg":"[1]R flush_numbers[0..12]: [293, 300, 296, 288, 289, 301, 294, 300, 296, 300, 291, 300]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:37.571764708Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1832,"":"downstairs","":"downstairs","session_id":"2a65786e-3920-4163-a002-f88b981c014a"} {"msg":"[1]R generation[0..12]: [167, 172, 170, 164, 164, 173, 168, 172, 170, 172, 166, 172]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:37.571775377Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1832,"":"downstairs","":"downstairs","session_id":"2a65786e-3920-4163-a002-f88b981c014a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:37.571786325Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1832,"":"downstairs","":"downstairs","session_id":"2a65786e-3920-4163-a002-f88b981c014a"} {"msg":"[2]R flush_numbers[0..12]: [293, 300, 296, 288, 289, 301, 294, 300, 296, 300, 291, 300]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:37.571796983Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1832,"":"downstairs","":"downstairs","session_id":"2a65786e-3920-4163-a002-f88b981c014a"} {"msg":"[2]R generation[0..12]: [167, 172, 170, 164, 164, 173, 168, 172, 170, 172, 166, 172]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:37.571807702Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1832,"":"downstairs","":"downstairs","session_id":"2a65786e-3920-4163-a002-f88b981c014a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:37.5718185Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1832,"":"downstairs","":"downstairs","session_id":"2a65786e-3920-4163-a002-f88b981c014a"} {"msg":"Max found gen is 174","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:37.571830059Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1832,"":"downstairs","":"downstairs","session_id":"2a65786e-3920-4163-a002-f88b981c014a"} {"msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:37.571839757Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1832,"":"downstairs","":"downstairs","session_id":"2a65786e-3920-4163-a002-f88b981c014a"} {"msg":"Next flush: 302","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:37.571849286Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1832,"":"downstairs","":"downstairs","session_id":"2a65786e-3920-4163-a002-f88b981c014a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:37.571858674Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1832,"":"downstairs","":"downstairs","session_id":"2a65786e-3920-4163-a002-f88b981c014a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:37.571868643Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1832,"session_id":"2a65786e-3920-4163-a002-f88b981c014a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:37.571877362Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1832,"client":"0","":"downstairs","session_id":"2a65786e-3920-4163-a002-f88b981c014a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:37.57188711Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1832,"client":"1","":"downstairs","session_id":"2a65786e-3920-4163-a002-f88b981c014a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:37.571896879Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1832,"client":"2","":"downstairs","session_id":"2a65786e-3920-4163-a002-f88b981c014a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:37.571906787Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1832,"session_id":"2a65786e-3920-4163-a002-f88b981c014a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:37.571915996Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1832,"session_id":"2a65786e-3920-4163-a002-f88b981c014a"} {"msg":"e1aae5dc-de37-42c8-8b54-5b8174202406 is now active with session: 2a65786e-3920-4163-a002-f88b981c014a","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:37.571925804Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1832,"session_id":"2a65786e-3920-4163-a002-f88b981c014a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:37.571934713Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1832,"session_id":"2a65786e-3920-4163-a002-f88b981c014a"} {"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:37.571943752Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1832} {"msg":"Activated sub_volume e1aae5dc-de37-42c8-8b54-5b8174202406","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:37.571951871Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1832} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:37.571959869Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1832,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:37.57229321Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1832,"task":"crutest"} 01/30 Write block 154 len 28672 data: 11 6 9 12 10 11 11 02/30 Write block 556 len 36864 data: 11 14 12 11 9 10 9 10 11 03/30 Read block 175 len 20480 04/30 Read block 97 len 32768 05/30 Read block 299 len 32768 06/30 Write block 252 len 28672 data: 14 16 19 17 19 20 22 07/30 Read block 178 len 32768 08/30 Read block 347 len 20480 09/30 Write block 241 len 20480 data: 11 9 9 11 11 10/30 Read block 457 len 20480 11/30 Read block 464 len 8192 12/30 Flush 13/30 Write block 149 len 20480 data: 14 13 14 14 15 14/30 Read block 579 len 20480 15/30 Read block 15 len 28672 16/30 Read block 80 len 32768 17/30 Write block 505 len 12288 data: 11 8 9 18/30 Read block 525 len 28672 19/30 Read block 540 len 16384 20/30 Flush 21/30 Write block 531 len 20480 data: 10 12 14 13 12 22/30 Write block 574 len 12288 data: 11 11 10 23/30 Read block 228 len 32768 24/30 Write block 265 len 8192 data: 11 14 25/30 Flush 26/30 Write block 421 len 4096 data: 8 27/30 Read block 51 len 20480 28/30 Read block 582 len 28672 29/30 Read block 149 len 24576 30/30 Write block 444 len 40960 data: 15 16 16 17 20 18 19 20 18 19 ---------------------------------------------------------------- Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Sent Done Done false 1001 Acked Write 9 Sent Done Done false 1002 Acked Read 5 Sent Done Done false 1003 Acked Read 8 Sent Done Done false 1004 Acked Read 8 Sent Done Done false 1005 Acked Write 7 Sent Done Done false 1006 Acked Read 8 Sent Done Done false 1007 Acked Read 5 Sent Done Done false 1008 Acked Write 5 Sent Done Done false 1009 Acked Read 5 Sent Done Done false 1010 Acked Read 2 Sent Done Done false 1011 Acked Flush 0 Sent Done Done false 1012 Acked Write 5 Sent Done Done false 1013 Acked Read 5 Sent Done Done false 1014 Acked Read 7 Sent Done Done false 1015 Acked Read 8 Sent Done Done false 1016 Acked Write 3 Sent Done Done false 1017 Acked Read 7 Sent Done Done false 1018 Acked Read 4 Sent Done Done false 1019 Acked Flush 0 Sent Done Done false 1020 Acked Write 5 Sent Done Done false 1021 Acked Write 3 Sent Done Done false 1022 Acked Read 8 Sent Done Done false 1023 Acked Write 2 Sent Done Done false 1024 Acked Flush 0 Sent Done Done false 1025 Acked Write 1 Sent Done Done false 1026 Acked Read 5 Sent Done Done false 1027 Acked Read 7 Sent Done Done false 1028 Acked Read 6 Sent Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 1 1 32 Done 0 29 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:59:37.581549085Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1832} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:37.581735557Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1832,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 293 293 293 F F F 1 020-039 172 172 172 300 300 300 F F F 2 040-059 170 170 170 296 296 296 F F F 3 060-079 164 164 164 288 288 288 F F F 4 080-099 164 164 164 289 289 289 F F F 5 100-119 173 173 173 301 301 301 F F F 6 120-139 168 168 168 294 294 294 F F F 7 140-159 172 174 174 300 303 303 F F F <--- 8 160-179 170 174 174 296 302 302 F F F <--- 9 180-199 172 172 172 300 300 300 F F F 10 200-219 166 166 166 291 291 291 F F F 11 220-239 172 172 172 300 300 300 F F F 12 240-259 172 174 174 300 302 302 F F F <--- 13 260-279 172 174 174 300 304 304 F F F <--- 14 280-299 172 172 172 300 300 300 F F F 15 300-319 168 168 168 294 294 294 F F F 16 320-339 168 168 168 294 294 294 F F F 17 340-359 172 172 172 300 300 300 F F F 18 360-379 173 173 173 301 301 301 F F F 19 380-399 173 173 173 301 301 301 F F F 20 400-419 164 164 164 289 289 289 F F F 21 420-439 166 166 166 291 291 291 F T T <--- 22 440-459 173 173 173 301 301 301 F T T <--- 23 460-479 168 168 168 294 294 294 F F F 24 480-499 172 172 172 300 300 300 F F F 25 500-519 171 174 174 298 303 303 F F F <--- 26 520-539 172 174 174 300 304 304 F F F <--- 27 540-559 172 174 174 300 302 302 F F F <--- 28 560-579 170 174 174 296 304 304 F F F <--- 29 580-599 168 168 168 294 294 294 F F F Max gen: 174, Max flush: 304 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 87 Verifying data now No change after verify Loop: 87 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 293 293 293 F F F 1 020-039 172 172 172 300 300 300 F F F 2 040-059 170 170 170 296 296 296 F F F 3 060-079 164 164 164 288 288 288 F F F 4 080-099 164 164 164 289 289 289 F F F 5 100-119 173 173 173 301 301 301 F F F 6 120-139 168 168 168 294 294 294 F F F 7 140-159 174 174 174 303 303 303 F F F 8 160-179 174 174 174 302 302 302 F F F 9 180-199 172 172 172 300 300 300 F F F 10 200-219 166 166 166 291 291 291 F F F 11 220-239 172 172 172 300 300 300 F F F 12 240-259 174 174 174 302 302 302 F F F 13 260-279 174 174 174 304 304 304 F F F 14 280-299 172 172 172 300 300 300 F F F 15 300-319 168 168 168 294 294 294 F F F 16 320-339 168 168 168 294 294 294 F F F 17 340-359 172 172 172 300 300 300 F F F 18 360-379 173 173 173 301 301 301 F F F 19 380-399 173 173 173 301 301 301 F F F 20 400-419 164 164 164 289 289 289 F F F 21 420-439 175 175 175 305 305 305 F F F 22 440-459 175 175 175 305 305 305 F F F 23 460-479 168 168 168 294 294 294 F F F 24 480-499 172 172 172 300 300 300 F F F 25 500-519 174 174 174 303 303 303 F F F 26 520-539 174 174 174 304 304 304 F F F 27 540-559 174 174 174 302 302 302 F F F 28 560-579 174 174 174 304 304 304 F F F 29 580-599 168 168 168 294 294 294 F F F Max gen: 175, Max flush: 305 Begin loop 88 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:41.830355734Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1840,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:59:41.837440148Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1840,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:41.837491531Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1840,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:41.844239965Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1840,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:41.844504186Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1840} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:41.844533522Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1840} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:41.844542111Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1840} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:41.844553519Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1840} {"msg":"Crucible 4a80578d-5b16-467d-9149-3ade1649b7e5 has session id: b53b5c13-2b04-4c9a-889f-c4b4c25da9ce","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:41.844834477Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1840,"session_id":"b53b5c13-2b04-4c9a-889f-c4b4c25da9ce"} {"msg":"Upstairs opts: Upstairs UUID: 4a80578d-5b16-467d-9149-3ade1649b7e5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:41.844864343Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1840,"session_id":"b53b5c13-2b04-4c9a-889f-c4b4c25da9ce"} {"msg":"Crucible stats registered with UUID: 4a80578d-5b16-467d-9149-3ade1649b7e5","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:41.844875611Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1840,"session_id":"b53b5c13-2b04-4c9a-889f-c4b4c25da9ce"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:59:41.84488496Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1840,"session_id":"b53b5c13-2b04-4c9a-889f-c4b4c25da9ce"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:41.845034678Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1840,"":"io task","client":"1","":"downstairs","session_id":"b53b5c13-2b04-4c9a-889f-c4b4c25da9ce"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:41.845066663Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1840,"":"io task","client":"0","":"downstairs","session_id":"b53b5c13-2b04-4c9a-889f-c4b4c25da9ce"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:41.845077702Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1840,"":"io task","client":"2","":"downstairs","session_id":"b53b5c13-2b04-4c9a-889f-c4b4c25da9ce"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:41.845228579Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1840,"task":"crutest"} {"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:41.845252746Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1840} {"msg":"4a80578d-5b16-467d-9149-3ade1649b7e5 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:41.845261904Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1840,"session_id":"b53b5c13-2b04-4c9a-889f-c4b4c25da9ce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:41.845270923Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1840,"client":"0","":"downstairs","session_id":"b53b5c13-2b04-4c9a-889f-c4b4c25da9ce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:41.845281062Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1840,"client":"1","":"downstairs","session_id":"b53b5c13-2b04-4c9a-889f-c4b4c25da9ce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:41.84529087Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1840,"client":"2","":"downstairs","session_id":"b53b5c13-2b04-4c9a-889f-c4b4c25da9ce"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:41.845300909Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1840,"":"io task","client":"2","":"downstairs","session_id":"b53b5c13-2b04-4c9a-889f-c4b4c25da9ce"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:41.845311037Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1840,"":"io task","client":"0","":"downstairs","session_id":"b53b5c13-2b04-4c9a-889f-c4b4c25da9ce"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:41.845321056Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1840,"":"io task","client":"1","":"downstairs","session_id":"b53b5c13-2b04-4c9a-889f-c4b4c25da9ce"} {"msg":"ds_connection connected from Ok(127.0.0.1:62758)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:41.845331514Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1840,"":"io task","client":"2","":"downstairs","session_id":"b53b5c13-2b04-4c9a-889f-c4b4c25da9ce"} {"msg":"ds_connection connected from Ok(127.0.0.1:57885)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:41.845342203Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1840,"":"io task","client":"0","":"downstairs","session_id":"b53b5c13-2b04-4c9a-889f-c4b4c25da9ce"} {"msg":"ds_connection connected from Ok(127.0.0.1:50358)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:41.845392135Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1840,"":"io task","client":"1","":"downstairs","session_id":"b53b5c13-2b04-4c9a-889f-c4b4c25da9ce"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:41.845952313Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1840,"client":"1","":"downstairs","session_id":"b53b5c13-2b04-4c9a-889f-c4b4c25da9ce"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:41.846005675Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1840,"client":"0","":"downstairs","session_id":"b53b5c13-2b04-4c9a-889f-c4b4c25da9ce"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:41.846020352Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1840,"client":"2","":"downstairs","session_id":"b53b5c13-2b04-4c9a-889f-c4b4c25da9ce"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:41.846265836Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1840,"session_id":"b53b5c13-2b04-4c9a-889f-c4b4c25da9ce"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:41.846291642Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1840,"session_id":"b53b5c13-2b04-4c9a-889f-c4b4c25da9ce"} {"msg":"[0]R flush_numbers[0..12]: [293, 300, 296, 288, 289, 301, 294, 303, 302, 300, 291, 300]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:41.846338666Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1840,"":"downstairs","":"downstairs","session_id":"b53b5c13-2b04-4c9a-889f-c4b4c25da9ce"} {"msg":"[0]R generation[0..12]: [167, 172, 170, 164, 164, 173, 168, 174, 174, 172, 166, 172]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:41.846350574Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1840,"":"downstairs","":"downstairs","session_id":"b53b5c13-2b04-4c9a-889f-c4b4c25da9ce"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:41.846361632Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1840,"":"downstairs","":"downstairs","session_id":"b53b5c13-2b04-4c9a-889f-c4b4c25da9ce"} {"msg":"[1]R flush_numbers[0..12]: [293, 300, 296, 288, 289, 301, 294, 303, 302, 300, 291, 300]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:41.846372471Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1840,"":"downstairs","":"downstairs","session_id":"b53b5c13-2b04-4c9a-889f-c4b4c25da9ce"} {"msg":"[1]R generation[0..12]: [167, 172, 170, 164, 164, 173, 168, 174, 174, 172, 166, 172]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:41.846383119Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1840,"":"downstairs","":"downstairs","session_id":"b53b5c13-2b04-4c9a-889f-c4b4c25da9ce"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:41.846394097Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1840,"":"downstairs","":"downstairs","session_id":"b53b5c13-2b04-4c9a-889f-c4b4c25da9ce"} {"msg":"[2]R flush_numbers[0..12]: [293, 300, 296, 288, 289, 301, 294, 303, 302, 300, 291, 300]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:41.846404586Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1840,"":"downstairs","":"downstairs","session_id":"b53b5c13-2b04-4c9a-889f-c4b4c25da9ce"} {"msg":"[2]R generation[0..12]: [167, 172, 170, 164, 164, 173, 168, 174, 174, 172, 166, 172]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:41.846415354Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1840,"":"downstairs","":"downstairs","session_id":"b53b5c13-2b04-4c9a-889f-c4b4c25da9ce"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:41.846426103Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1840,"":"downstairs","":"downstairs","session_id":"b53b5c13-2b04-4c9a-889f-c4b4c25da9ce"} {"msg":"Max found gen is 176","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:41.846435581Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1840,"":"downstairs","":"downstairs","session_id":"b53b5c13-2b04-4c9a-889f-c4b4c25da9ce"} {"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:41.84644526Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1840,"":"downstairs","":"downstairs","session_id":"b53b5c13-2b04-4c9a-889f-c4b4c25da9ce"} {"msg":"Next flush: 306","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:41.846454868Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1840,"":"downstairs","":"downstairs","session_id":"b53b5c13-2b04-4c9a-889f-c4b4c25da9ce"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:41.846464497Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1840,"":"downstairs","":"downstairs","session_id":"b53b5c13-2b04-4c9a-889f-c4b4c25da9ce"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:41.846475535Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1840,"session_id":"b53b5c13-2b04-4c9a-889f-c4b4c25da9ce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:41.846484294Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1840,"client":"0","":"downstairs","session_id":"b53b5c13-2b04-4c9a-889f-c4b4c25da9ce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:41.846499182Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1840,"client":"1","":"downstairs","session_id":"b53b5c13-2b04-4c9a-889f-c4b4c25da9ce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:41.84650881Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1840,"client":"2","":"downstairs","session_id":"b53b5c13-2b04-4c9a-889f-c4b4c25da9ce"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:41.846518709Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1840,"session_id":"b53b5c13-2b04-4c9a-889f-c4b4c25da9ce"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:41.846527948Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1840,"session_id":"b53b5c13-2b04-4c9a-889f-c4b4c25da9ce"} {"msg":"4a80578d-5b16-467d-9149-3ade1649b7e5 is now active with session: b53b5c13-2b04-4c9a-889f-c4b4c25da9ce","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:41.846537916Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1840,"session_id":"b53b5c13-2b04-4c9a-889f-c4b4c25da9ce"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:41.846546715Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1840,"session_id":"b53b5c13-2b04-4c9a-889f-c4b4c25da9ce"} {"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:41.846555913Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1840} {"msg":"Activated sub_volume 4a80578d-5b16-467d-9149-3ade1649b7e5","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:41.846564012Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1840} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:41.846572051Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1840,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:41.84685263Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1840,"task":"crutest"} 01/30 Write block 257 len 32768 data: 21 23 20 13 15 13 15 14 02/30 Write block 160 len 24576 data: 12 11 10 10 11 13 03/30 Read block 12 len 24576 04/30 Read block 584 len 20480 05/30 Read block 369 len 4096 06/30 Write block 43 len 40960 data: 16 15 15 19 18 18 17 16 14 12 07/30 Write block 403 len 28672 data: 10 11 13 10 10 12 12 08/30 Flush 09/30 Write block 116 len 40960 data: 14 15 14 15 16 19 15 16 14 11 10/30 Read block 574 len 32768 11/30 Read block 161 len 8192 12/30 Read block 47 len 36864 13/30 Write block 196 len 40960 data: 16 17 15 15 13 9 10 13 14 15 14/30 Read block 390 len 28672 15/30 Flush 16/30 Write block 345 len 32768 data: 9 9 7 8 6 7 9 10 17/30 Read block 50 len 36864 18/30 Write block 567 len 28672 data: 7 8 10 12 12 11 11 19/30 Read block 7 len 40960 20/30 Write block 131 len 12288 data: 10 10 13 21/30 Write block 496 len 40960 data: 11 10 9 12 12 11 13 14 13 12 22/30 Read block 169 len 24576 23/30 Read block 319 len 24576 24/30 Flush 25/30 Write block 432 len 16384 data: 14 14 11 14 26/30 Write block 516 len 40960 data: 16 16 17 15 14 14 11 7 9 8 27/30 Write block 137 len 32768 data: 11 13 14 15 17 16 16 18 28/30 Read block 530 len 40960 29/30 Write block 357 len 24576 data: 14 13 11 13 9 9 30/30 Read block 390 len 40960 ---------------------------------------------------------------- Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Write 6 Done Done Done false 1002 Acked Read 6 Done Done Done false 1003 Acked Read 5 Done Done Done false 1004 Acked Read 1 Done Done Done false 1005 Acked Write 10 Done Done Done false 1006 Acked Write 7 Sent Done Done false 1007 Acked Flush 0 Sent Done Done false 1008 Acked Write 10 Sent Done Done false 1009 Acked Read 8 Sent Done Done false 1010 Acked Read 2 Sent Done Done false 1011 Acked Read 9 Sent Done Done false 1012 Acked Write 10 Sent Done Done false 1013 Acked Read 7 Sent Done Done false 1014 Acked Flush 0 Sent Done Done false 1015 Acked Write 8 Sent Done Done false 1016 Acked Read 9 Sent Done Done false 1017 Acked Write 7 Sent Done Done false 1018 Acked Read 10 Sent Done Done false 1019 Acked Write 3 Sent Done Done false 1020 Acked Write 10 Sent Done Done false 1021 Acked Read 6 Sent Done Done false 1022 Acked Read 6 Sent Done Done false 1023 Acked Flush 0 Sent Done Done false 1024 Acked Write 4 Sent Done Done false 1025 Acked Write 10 Sent Done Done false 1026 Acked Write 8 Sent Done Done false 1027 Acked Read 10 Sent Done Done false 1028 Acked Write 6 Sent Done Done false 1029 Acked Read 10 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 24 0 0 24 Done 6 30 30 66 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:59:41.857375737Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1840} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:41.857564889Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1840,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 293 293 293 F F F 1 020-039 172 172 172 300 300 300 F F F 2 040-059 170 176 176 296 306 306 T F F <--- 3 060-079 164 164 164 288 288 288 F F F 4 080-099 164 164 164 289 289 289 F F F 5 100-119 173 176 176 301 307 307 F F F <--- 6 120-139 168 176 176 294 308 308 F T T <--- 7 140-159 174 174 174 303 303 303 F T T <--- 8 160-179 174 176 176 302 306 306 T F F <--- 9 180-199 172 176 176 300 307 307 F F F <--- 10 200-219 166 176 176 291 307 307 F F F <--- 11 220-239 172 172 172 300 300 300 F F F 12 240-259 174 176 176 302 306 306 T F F <--- 13 260-279 174 176 176 304 306 306 T F F <--- 14 280-299 172 172 172 300 300 300 F F F 15 300-319 168 168 168 294 294 294 F F F 16 320-339 168 168 168 294 294 294 F F F 17 340-359 172 176 176 300 308 308 F T T <--- 18 360-379 173 173 173 301 301 301 F T T <--- 19 380-399 173 173 173 301 301 301 F F F 20 400-419 164 176 176 289 306 306 F F F <--- 21 420-439 175 175 175 305 305 305 F T T <--- 22 440-459 175 175 175 305 305 305 F F F 23 460-479 168 168 168 294 294 294 F F F 24 480-499 172 176 176 300 308 308 F F F <--- 25 500-519 174 176 176 303 308 308 F T T <--- 26 520-539 174 174 174 304 304 304 F T T <--- 27 540-559 174 174 174 302 302 302 F F F 28 560-579 174 176 176 304 308 308 F F F <--- 29 580-599 168 168 168 294 294 294 F F F Max gen: 176, Max flush: 308 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 88 Verifying data now No change after verify Loop: 88 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 293 293 293 F F F 1 020-039 172 172 172 300 300 300 F F F 2 040-059 176 176 176 306 306 306 F F F 3 060-079 164 164 164 288 288 288 F F F 4 080-099 164 164 164 289 289 289 F F F 5 100-119 176 176 176 307 307 307 F F F 6 120-139 177 177 177 309 309 309 F F F 7 140-159 177 177 177 309 309 309 F F F 8 160-179 176 176 176 306 306 306 F F F 9 180-199 176 176 176 307 307 307 F F F 10 200-219 176 176 176 307 307 307 F F F 11 220-239 172 172 172 300 300 300 F F F 12 240-259 176 176 176 306 306 306 F F F 13 260-279 176 176 176 306 306 306 F F F 14 280-299 172 172 172 300 300 300 F F F 15 300-319 168 168 168 294 294 294 F F F 16 320-339 168 168 168 294 294 294 F F F 17 340-359 177 177 177 309 309 309 F F F 18 360-379 177 177 177 309 309 309 F F F 19 380-399 173 173 173 301 301 301 F F F 20 400-419 176 176 176 306 306 306 F F F 21 420-439 177 177 177 309 309 309 F F F 22 440-459 175 175 175 305 305 305 F F F 23 460-479 168 168 168 294 294 294 F F F 24 480-499 176 176 176 308 308 308 F F F 25 500-519 177 177 177 309 309 309 F F F 26 520-539 177 177 177 309 309 309 F F F 27 540-559 174 174 174 302 302 302 F F F 28 560-579 176 176 176 308 308 308 F F F 29 580-599 168 168 168 294 294 294 F F F Max gen: 177, Max flush: 309 Begin loop 89 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:46.119741056Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1848,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:59:46.12683446Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1848,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:46.126884892Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1848,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:46.133633207Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1848,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:46.133898918Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1848} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:46.133924014Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1848} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:46.133932593Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1848} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:46.133944101Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1848} {"msg":"Crucible 7dee8033-d3f4-45a2-ac3c-845718630791 has session id: ab8020ef-7967-4999-ab31-79753c0799fc","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:46.134278302Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1848,"session_id":"ab8020ef-7967-4999-ab31-79753c0799fc"} {"msg":"Upstairs opts: Upstairs UUID: 7dee8033-d3f4-45a2-ac3c-845718630791, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:46.134307647Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1848,"session_id":"ab8020ef-7967-4999-ab31-79753c0799fc"} {"msg":"Crucible stats registered with UUID: 7dee8033-d3f4-45a2-ac3c-845718630791","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:46.134318866Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1848,"session_id":"ab8020ef-7967-4999-ab31-79753c0799fc"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:59:46.134328244Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1848,"session_id":"ab8020ef-7967-4999-ab31-79753c0799fc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:46.134460625Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1848,"":"io task","client":"1","":"downstairs","session_id":"ab8020ef-7967-4999-ab31-79753c0799fc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:46.13449069Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1848,"":"io task","client":"0","":"downstairs","session_id":"ab8020ef-7967-4999-ab31-79753c0799fc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:46.134501669Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1848,"":"io task","client":"2","":"downstairs","session_id":"ab8020ef-7967-4999-ab31-79753c0799fc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:46.134603114Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1848,"task":"crutest"} {"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:46.134618671Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1848} {"msg":"7dee8033-d3f4-45a2-ac3c-845718630791 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:46.134636519Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1848,"session_id":"ab8020ef-7967-4999-ab31-79753c0799fc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:46.134645447Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1848,"client":"0","":"downstairs","session_id":"ab8020ef-7967-4999-ab31-79753c0799fc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:46.134657226Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1848,"client":"1","":"downstairs","session_id":"ab8020ef-7967-4999-ab31-79753c0799fc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:46.134667154Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1848,"client":"2","":"downstairs","session_id":"ab8020ef-7967-4999-ab31-79753c0799fc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:46.134676843Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1848,"":"io task","client":"2","":"downstairs","session_id":"ab8020ef-7967-4999-ab31-79753c0799fc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:46.134687001Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1848,"":"io task","client":"0","":"downstairs","session_id":"ab8020ef-7967-4999-ab31-79753c0799fc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:46.13469693Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1848,"":"io task","client":"1","":"downstairs","session_id":"ab8020ef-7967-4999-ab31-79753c0799fc"} {"msg":"ds_connection connected from Ok(127.0.0.1:54667)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:46.13483184Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1848,"":"io task","client":"2","":"downstairs","session_id":"ab8020ef-7967-4999-ab31-79753c0799fc"} {"msg":"ds_connection connected from Ok(127.0.0.1:45022)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:46.134859566Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1848,"":"io task","client":"1","":"downstairs","session_id":"ab8020ef-7967-4999-ab31-79753c0799fc"} {"msg":"ds_connection connected from Ok(127.0.0.1:49074)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:46.134871474Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1848,"":"io task","client":"0","":"downstairs","session_id":"ab8020ef-7967-4999-ab31-79753c0799fc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:46.135553184Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1848,"client":"1","":"downstairs","session_id":"ab8020ef-7967-4999-ab31-79753c0799fc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:46.135594747Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1848,"client":"0","":"downstairs","session_id":"ab8020ef-7967-4999-ab31-79753c0799fc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:46.135620984Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1848,"client":"2","":"downstairs","session_id":"ab8020ef-7967-4999-ab31-79753c0799fc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:46.135843981Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1848,"session_id":"ab8020ef-7967-4999-ab31-79753c0799fc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:46.135869787Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1848,"session_id":"ab8020ef-7967-4999-ab31-79753c0799fc"} {"msg":"[0]R flush_numbers[0..12]: [293, 300, 306, 288, 289, 307, 309, 309, 306, 307, 307, 300]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:46.135929538Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1848,"":"downstairs","":"downstairs","session_id":"ab8020ef-7967-4999-ab31-79753c0799fc"} {"msg":"[0]R generation[0..12]: [167, 172, 176, 164, 164, 176, 177, 177, 176, 176, 176, 172]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:46.135940936Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1848,"":"downstairs","":"downstairs","session_id":"ab8020ef-7967-4999-ab31-79753c0799fc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:46.135954124Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1848,"":"downstairs","":"downstairs","session_id":"ab8020ef-7967-4999-ab31-79753c0799fc"} {"msg":"[1]R flush_numbers[0..12]: [293, 300, 306, 288, 289, 307, 309, 309, 306, 307, 307, 300]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:46.135964843Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1848,"":"downstairs","":"downstairs","session_id":"ab8020ef-7967-4999-ab31-79753c0799fc"} {"msg":"[1]R generation[0..12]: [167, 172, 176, 164, 164, 176, 177, 177, 176, 176, 176, 172]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:46.135975411Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1848,"":"downstairs","":"downstairs","session_id":"ab8020ef-7967-4999-ab31-79753c0799fc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:46.13598624Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1848,"":"downstairs","":"downstairs","session_id":"ab8020ef-7967-4999-ab31-79753c0799fc"} {"msg":"[2]R flush_numbers[0..12]: [293, 300, 306, 288, 289, 307, 309, 309, 306, 307, 307, 300]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:46.135997178Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1848,"":"downstairs","":"downstairs","session_id":"ab8020ef-7967-4999-ab31-79753c0799fc"} {"msg":"[2]R generation[0..12]: [167, 172, 176, 164, 164, 176, 177, 177, 176, 176, 176, 172]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:46.136007667Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1848,"":"downstairs","":"downstairs","session_id":"ab8020ef-7967-4999-ab31-79753c0799fc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:46.136018815Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1848,"":"downstairs","":"downstairs","session_id":"ab8020ef-7967-4999-ab31-79753c0799fc"} {"msg":"Max found gen is 178","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:46.136028153Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1848,"":"downstairs","":"downstairs","session_id":"ab8020ef-7967-4999-ab31-79753c0799fc"} {"msg":"Generation requested: 178 >= found:178","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:46.136038002Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1848,"":"downstairs","":"downstairs","session_id":"ab8020ef-7967-4999-ab31-79753c0799fc"} {"msg":"Next flush: 310","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:46.136047461Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1848,"":"downstairs","":"downstairs","session_id":"ab8020ef-7967-4999-ab31-79753c0799fc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:46.136056799Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1848,"":"downstairs","":"downstairs","session_id":"ab8020ef-7967-4999-ab31-79753c0799fc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:46.136066888Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1848,"session_id":"ab8020ef-7967-4999-ab31-79753c0799fc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:46.136075596Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1848,"client":"0","":"downstairs","session_id":"ab8020ef-7967-4999-ab31-79753c0799fc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:46.136085535Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1848,"client":"1","":"downstairs","session_id":"ab8020ef-7967-4999-ab31-79753c0799fc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:46.136096753Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1848,"client":"2","":"downstairs","session_id":"ab8020ef-7967-4999-ab31-79753c0799fc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:46.136106822Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1848,"session_id":"ab8020ef-7967-4999-ab31-79753c0799fc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:46.136116021Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1848,"session_id":"ab8020ef-7967-4999-ab31-79753c0799fc"} {"msg":"7dee8033-d3f4-45a2-ac3c-845718630791 is now active with session: ab8020ef-7967-4999-ab31-79753c0799fc","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:46.136125999Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1848,"session_id":"ab8020ef-7967-4999-ab31-79753c0799fc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:46.136134698Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1848,"session_id":"ab8020ef-7967-4999-ab31-79753c0799fc"} {"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:46.136143746Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1848} {"msg":"Activated sub_volume 7dee8033-d3f4-45a2-ac3c-845718630791","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:46.136151625Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1848} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:46.136159664Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1848,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:46.136441473Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1848,"task":"crutest"} 01/30 Write block 223 len 32768 data: 12 14 11 10 11 13 15 15 02/30 Flush 03/30 Write block 324 len 28672 data: 17 18 19 15 14 15 16 04/30 Write block 531 len 12288 data: 11 13 15 05/30 Read block 368 len 4096 06/30 Write block 503 len 4096 data: 15 07/30 Write block 448 len 16384 data: 21 19 20 21 08/30 Write block 479 len 8192 data: 18 18 09/30 Write block 85 len 40960 data: 7 10 11 16 16 16 14 15 15 12 10/30 Write block 402 len 8192 data: 6 11 11/30 Write block 161 len 20480 data: 12 11 11 12 14 12/30 Write block 414 len 36864 data: 15 16 17 12 10 10 9 9 11 13/30 Read block 276 len 16384 14/30 Read block 295 len 20480 15/30 Read block 357 len 16384 16/30 Write block 585 len 24576 data: 15 15 17 14 13 14 17/30 Write block 107 len 28672 data: 13 12 9 9 13 13 12 18/30 Write block 63 len 36864 data: 11 11 10 11 14 12 11 10 11 19/30 Read block 117 len 28672 20/30 Write block 415 len 40960 data: 17 18 13 11 11 10 10 12 13 16 21/30 Write block 127 len 40960 data: 10 9 11 9 11 11 14 16 15 13 22/30 Read block 299 len 20480 23/30 Read block 520 len 28672 24/30 Read block 299 len 4096 25/30 Flush 26/30 Write block 140 len 28672 data: 16 18 17 17 19 17 17 27/30 Write block 208 len 28672 data: 20 18 15 11 13 13 13 28/30 Read block 465 len 20480 29/30 Read block 155 len 8192 30/30 Write block 485 len 36864 data: 13 9 11 15 13 11 7 8 5 ---------------------------------------------------------------- Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 7 Done Done Done false 1003 Acked Write 3 Done Done Done false 1004 Acked Read 1 Done Done Done false 1005 Acked Write 1 Done Done Done false 1006 Acked Write 4 Done Done Done false 1007 Acked Write 2 Done Done Done false 1008 Acked Write 10 Done Done Done false 1009 Acked Write 2 Sent Done Done false 1010 Acked Write 5 Sent Done Done false 1011 Acked Write 9 Sent Done Done false 1012 Acked Read 4 Sent Done Done false 1013 Acked Read 5 Sent Done Done false 1014 Acked Read 4 Sent Done Done false 1015 Acked Write 6 Sent Done Done false 1016 Acked Write 7 Sent Done Done false 1017 Acked Write 9 Sent Done Done false 1018 Acked Read 7 Sent Done Done false 1019 Acked Write 10 Sent Done Done false 1020 Acked Write 10 Sent Done Done false 1021 Acked Read 5 Sent Done Done false 1022 Acked Read 7 Sent Done Done false 1023 Acked Read 1 Sent Done Done false 1024 Acked Flush 0 Sent Done Done false 1025 Acked Write 7 Sent Done Done false 1026 Acked Write 7 Sent Done Done false 1027 Acked Read 5 Sent Done Done false 1028 Acked Read 2 Sent Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 21 1 1 23 Done 7 27 27 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1001)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:59:46.145209409Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1848} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:46.145384813Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1848,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 293 293 293 F F F 1 020-039 172 172 172 300 300 300 F F F 2 040-059 176 176 176 306 306 306 F F F 3 060-079 164 178 178 288 311 311 F F F <--- 4 080-099 164 178 178 289 311 311 T F F <--- 5 100-119 176 178 178 307 311 311 F F F <--- 6 120-139 177 178 178 309 311 311 F F F <--- 7 140-159 177 177 177 309 309 309 F T T <--- 8 160-179 176 178 178 306 311 311 F F F <--- 9 180-199 176 176 176 307 307 307 F F F 10 200-219 176 176 176 307 307 307 F T T <--- 11 220-239 178 178 178 310 310 310 F F F 12 240-259 176 176 176 306 306 306 F F F 13 260-279 176 176 176 306 306 306 F F F 14 280-299 172 172 172 300 300 300 F F F 15 300-319 168 168 168 294 294 294 F F F 16 320-339 168 178 178 294 311 311 T F F <--- 17 340-359 177 177 177 309 309 309 F F F 18 360-379 177 177 177 309 309 309 F F F 19 380-399 173 173 173 301 301 301 F F F 20 400-419 176 178 178 306 311 311 F F F <--- 21 420-439 177 178 178 309 311 311 F F F <--- 22 440-459 175 178 178 305 311 311 T F F <--- 23 460-479 168 178 178 294 311 311 T F F <--- 24 480-499 176 178 178 308 311 311 T T T <--- 25 500-519 177 178 178 309 311 311 T F F <--- 26 520-539 177 178 178 309 311 311 T F F <--- 27 540-559 174 174 174 302 302 302 F F F 28 560-579 176 176 176 308 308 308 F F F 29 580-599 168 178 178 294 311 311 F F F <--- Max gen: 178, Max flush: 311 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 89 Verifying data now No change after verify Loop: 89 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 293 293 293 F F F 1 020-039 172 172 172 300 300 300 F F F 2 040-059 176 176 176 306 306 306 F F F 3 060-079 178 178 178 311 311 311 F F F 4 080-099 178 178 178 311 311 311 F F F 5 100-119 178 178 178 311 311 311 F F F 6 120-139 178 178 178 311 311 311 F F F 7 140-159 179 179 179 312 312 312 F F F 8 160-179 178 178 178 311 311 311 F F F 9 180-199 176 176 176 307 307 307 F F F 10 200-219 179 179 179 312 312 312 F F F 11 220-239 178 178 178 310 310 310 F F F 12 240-259 176 176 176 306 306 306 F F F 13 260-279 176 176 176 306 306 306 F F F 14 280-299 172 172 172 300 300 300 F F F 15 300-319 168 168 168 294 294 294 F F F 16 320-339 178 178 178 311 311 311 F F F 17 340-359 177 177 177 309 309 309 F F F 18 360-379 177 177 177 309 309 309 F F F 19 380-399 173 173 173 301 301 301 F F F 20 400-419 178 178 178 311 311 311 F F F 21 420-439 178 178 178 311 311 311 F F F 22 440-459 178 178 178 311 311 311 F F F 23 460-479 178 178 178 311 311 311 F F F 24 480-499 179 179 179 312 312 312 F F F 25 500-519 178 178 178 311 311 311 F F F 26 520-539 178 178 178 311 311 311 F F F 27 540-559 174 174 174 302 302 302 F F F 28 560-579 176 176 176 308 308 308 F F F 29 580-599 178 178 178 311 311 311 F F F Max gen: 179, Max flush: 312 Begin loop 90 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:50.40464833Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1856,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:50.413068718Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1856,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:50.413419386Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1856} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:50.413443862Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1856} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:50.413452311Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1856} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:50.413463639Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1856} {"msg":"Crucible 1347a665-4297-4755-a78c-8da2dbb82841 has session id: 2bf46b47-d848-494d-8d8c-25f68c841239","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:50.413802149Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1856,"session_id":"2bf46b47-d848-494d-8d8c-25f68c841239"} {"msg":"Upstairs opts: Upstairs UUID: 1347a665-4297-4755-a78c-8da2dbb82841, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:50.413823186Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1856,"session_id":"2bf46b47-d848-494d-8d8c-25f68c841239"} {"msg":"Crucible stats registered with UUID: 1347a665-4297-4755-a78c-8da2dbb82841","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:50.413833185Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1856,"session_id":"2bf46b47-d848-494d-8d8c-25f68c841239"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:59:50.413842034Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1856,"session_id":"2bf46b47-d848-494d-8d8c-25f68c841239"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:50.413990972Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1856,"":"io task","client":"1","":"downstairs","session_id":"2bf46b47-d848-494d-8d8c-25f68c841239"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:50.414022207Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1856,"":"io task","client":"0","":"downstairs","session_id":"2bf46b47-d848-494d-8d8c-25f68c841239"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:50.414033365Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1856,"":"io task","client":"2","":"downstairs","session_id":"2bf46b47-d848-494d-8d8c-25f68c841239"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:50.414159807Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1856,"task":"crutest"} {"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:50.414174065Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1856} {"msg":"1347a665-4297-4755-a78c-8da2dbb82841 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:50.414201031Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1856,"session_id":"2bf46b47-d848-494d-8d8c-25f68c841239"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:50.414210649Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1856,"client":"0","":"downstairs","session_id":"2bf46b47-d848-494d-8d8c-25f68c841239"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:50.414220338Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1856,"client":"1","":"downstairs","session_id":"2bf46b47-d848-494d-8d8c-25f68c841239"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:50.414229966Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1856,"client":"2","":"downstairs","session_id":"2bf46b47-d848-494d-8d8c-25f68c841239"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:50.414239565Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1856,"":"io task","client":"2","":"downstairs","session_id":"2bf46b47-d848-494d-8d8c-25f68c841239"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:50.414249763Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1856,"":"io task","client":"0","":"downstairs","session_id":"2bf46b47-d848-494d-8d8c-25f68c841239"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:50.414259582Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1856,"":"io task","client":"1","":"downstairs","session_id":"2bf46b47-d848-494d-8d8c-25f68c841239"} {"msg":"ds_connection connected from Ok(127.0.0.1:49564)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:50.414330562Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1856,"":"io task","client":"2","":"downstairs","session_id":"2bf46b47-d848-494d-8d8c-25f68c841239"} {"msg":"ds_connection connected from Ok(127.0.0.1:62990)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:50.41434161Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1856,"":"io task","client":"1","":"downstairs","session_id":"2bf46b47-d848-494d-8d8c-25f68c841239"} {"msg":"ds_connection connected from Ok(127.0.0.1:60738)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:50.414352178Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1856,"":"io task","client":"0","":"downstairs","session_id":"2bf46b47-d848-494d-8d8c-25f68c841239"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:50.414940522Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1856,"client":"1","":"downstairs","session_id":"2bf46b47-d848-494d-8d8c-25f68c841239"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:50.414970667Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1856,"client":"0","":"downstairs","session_id":"2bf46b47-d848-494d-8d8c-25f68c841239"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:50.415147311Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1856,"client":"2","":"downstairs","session_id":"2bf46b47-d848-494d-8d8c-25f68c841239"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:50.415241327Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1856,"session_id":"2bf46b47-d848-494d-8d8c-25f68c841239"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:50.415266413Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1856,"session_id":"2bf46b47-d848-494d-8d8c-25f68c841239"} {"msg":"[0]R flush_numbers[0..12]: [293, 300, 306, 311, 311, 311, 311, 312, 311, 307, 312, 310]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:50.415440388Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1856,"":"downstairs","":"downstairs","session_id":"2bf46b47-d848-494d-8d8c-25f68c841239"} {"msg":"[0]R generation[0..12]: [167, 172, 176, 178, 178, 178, 178, 179, 178, 176, 179, 178]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:50.415451826Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1856,"":"downstairs","":"downstairs","session_id":"2bf46b47-d848-494d-8d8c-25f68c841239"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:50.415462674Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1856,"":"downstairs","":"downstairs","session_id":"2bf46b47-d848-494d-8d8c-25f68c841239"} {"msg":"[1]R flush_numbers[0..12]: [293, 300, 306, 311, 311, 311, 311, 312, 311, 307, 312, 310]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:50.415473223Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1856,"":"downstairs","":"downstairs","session_id":"2bf46b47-d848-494d-8d8c-25f68c841239"} {"msg":"[1]R generation[0..12]: [167, 172, 176, 178, 178, 178, 178, 179, 178, 176, 179, 178]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:50.415483821Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1856,"":"downstairs","":"downstairs","session_id":"2bf46b47-d848-494d-8d8c-25f68c841239"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:50.41549472Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1856,"":"downstairs","":"downstairs","session_id":"2bf46b47-d848-494d-8d8c-25f68c841239"} {"msg":"[2]R flush_numbers[0..12]: [293, 300, 306, 311, 311, 311, 311, 312, 311, 307, 312, 310]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:50.415508578Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1856,"":"downstairs","":"downstairs","session_id":"2bf46b47-d848-494d-8d8c-25f68c841239"} {"msg":"[2]R generation[0..12]: [167, 172, 176, 178, 178, 178, 178, 179, 178, 176, 179, 178]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:50.415519166Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1856,"":"downstairs","":"downstairs","session_id":"2bf46b47-d848-494d-8d8c-25f68c841239"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:50.415530125Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1856,"":"downstairs","":"downstairs","session_id":"2bf46b47-d848-494d-8d8c-25f68c841239"} {"msg":"Max found gen is 180","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:50.415539573Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1856,"":"downstairs","":"downstairs","session_id":"2bf46b47-d848-494d-8d8c-25f68c841239"} {"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:50.415549412Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1856,"":"downstairs","":"downstairs","session_id":"2bf46b47-d848-494d-8d8c-25f68c841239"} {"msg":"Next flush: 313","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:50.41555883Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1856,"":"downstairs","":"downstairs","session_id":"2bf46b47-d848-494d-8d8c-25f68c841239"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:50.415568239Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1856,"":"downstairs","":"downstairs","session_id":"2bf46b47-d848-494d-8d8c-25f68c841239"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:50.415577897Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1856,"session_id":"2bf46b47-d848-494d-8d8c-25f68c841239"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:50.415587026Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1856,"client":"0","":"downstairs","session_id":"2bf46b47-d848-494d-8d8c-25f68c841239"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:50.415596625Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1856,"client":"1","":"downstairs","session_id":"2bf46b47-d848-494d-8d8c-25f68c841239"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:50.415606353Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1856,"client":"2","":"downstairs","session_id":"2bf46b47-d848-494d-8d8c-25f68c841239"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:50.415616262Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1856,"session_id":"2bf46b47-d848-494d-8d8c-25f68c841239"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:50.41562531Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1856,"session_id":"2bf46b47-d848-494d-8d8c-25f68c841239"} {"msg":"1347a665-4297-4755-a78c-8da2dbb82841 is now active with session: 2bf46b47-d848-494d-8d8c-25f68c841239","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:50.415635349Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1856,"session_id":"2bf46b47-d848-494d-8d8c-25f68c841239"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:50.415643998Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1856,"session_id":"2bf46b47-d848-494d-8d8c-25f68c841239"} {"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:50.415654716Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1856} {"msg":"Activated sub_volume 1347a665-4297-4755-a78c-8da2dbb82841","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:50.415662635Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1856} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:50.415670694Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1856,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:50.415991606Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1856,"task":"crutest"} 01/30 Flush 02/30 Write block 368 len 40960 data: 15 14 15 13 15 14 13 13 12 12 03/30 Write block 182 len 36864 data: 11 11 12 9 9 12 12 11 11 04/30 Write block 562 len 20480 data: 10 11 12 10 7 05/30 Write block 304 len 4096 data: 6 06/30 Write block 140 len 16384 data: 17 19 18 18 07/30 Read block 12 len 40960 08/30 Write block 562 len 32768 data: 11 12 13 11 8 8 9 11 09/30 Write block 498 len 36864 data: 10 13 13 12 14 16 14 13 9 10/30 Read block 370 len 40960 11/30 Read block 309 len 16384 12/30 Read block 442 len 8192 13/30 Write block 261 len 8192 data: 16 14 14/30 Read block 350 len 24576 15/30 Read block 96 len 4096 16/30 Write block 553 len 4096 data: 16 17/30 Read block 302 len 8192 18/30 Write block 35 len 24576 data: 16 16 14 16 19 18 19/30 Write block 289 len 20480 data: 20 20 15 13 12 20/30 Write block 485 len 4096 data: 14 21/30 Read block 202 len 28672 22/30 Flush 23/30 Write block 172 len 32768 data: 12 13 13 14 16 17 15 11 24/30 Read block 445 len 36864 25/30 Flush 26/30 Write block 339 len 16384 data: 15 14 13 13 27/30 Read block 2 len 40960 28/30 Read block 341 len 8192 29/30 Write block 221 len 32768 data: 13 14 13 15 12 11 12 14 30/30 Read block 405 len 16384 ---------------------------------------------------------------- Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Flush 0 Done Done Sent false 1001 Acked Write 10 Done Done Sent false 1002 Acked Write 9 Done Done Sent false 1003 Acked Write 5 Done Done Sent false 1004 Acked Write 1 Done Done Sent false 1005 Acked Write 4 Done Done Sent false 1006 Acked Read 10 Done Done Sent false 1007 Acked Write 8 Done Done Sent false 1008 Acked Write 9 Done Done Sent false 1009 Acked Read 10 Done Done Sent false 1010 Acked Read 4 Done Done Sent false 1011 Acked Read 2 Done Done Sent false 1012 Acked Write 2 Done Done Sent false 1013 Acked Read 6 Done Done Sent false 1014 Acked Read 1 Done Done Sent false 1015 Acked Write 1 Done Done Sent false 1016 Acked Read 2 Done Done Sent false 1017 Acked Write 6 Done Done Sent false 1018 Acked Write 5 Done Done Sent false 1019 Acked Write 1 Done Done Sent false 1020 Acked Read 7 Done Done Sent false 1021 Acked Flush 0 Done Done Sent false 1022 Acked Write 8 Done Done Sent false 1023 Acked Read 9 Done Done Sent false 1024 Acked Flush 0 Done Done Sent false 1025 Acked Write 4 Done Done Sent false 1026 Acked Read 10 Done Done Sent false 1027 Acked Read 2 Done Done Sent false 1028 Acked Write 8 Done Done Sent false 1029 Acked Read 4 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 30 30 Done 30 30 0 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:59:50.424545604Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1856} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:50.424726378Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1856,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 293 293 293 F F F 1 020-039 180 180 172 314 314 300 F F F <--- 2 040-059 180 180 176 314 314 306 F F F <--- 3 060-079 178 178 178 311 311 311 F F F 4 080-099 178 178 178 311 311 311 F F F 5 100-119 178 178 178 311 311 311 F F F 6 120-139 178 178 178 311 311 311 F F F 7 140-159 180 180 179 314 314 312 F F F <--- 8 160-179 180 180 178 315 315 311 F F F <--- 9 180-199 180 180 176 314 314 307 F F F <--- 10 200-219 179 179 179 312 312 312 F F F 11 220-239 178 178 178 310 310 310 T T F <--- 12 240-259 176 176 176 306 306 306 F F F 13 260-279 180 180 176 314 314 306 F F F <--- 14 280-299 180 180 172 314 314 300 F F F <--- 15 300-319 180 180 168 314 314 294 F F F <--- 16 320-339 178 178 178 311 311 311 T T F <--- 17 340-359 177 177 177 309 309 309 T T F <--- 18 360-379 180 180 177 314 314 309 F F F <--- 19 380-399 173 173 173 301 301 301 F F F 20 400-419 178 178 178 311 311 311 F F F 21 420-439 178 178 178 311 311 311 F F F 22 440-459 178 178 178 311 311 311 F F F 23 460-479 178 178 178 311 311 311 F F F 24 480-499 180 180 179 314 314 312 F F F <--- 25 500-519 180 180 178 314 314 311 F F F <--- 26 520-539 178 178 178 311 311 311 F F F 27 540-559 180 180 174 314 314 302 F F F <--- 28 560-579 180 180 176 314 314 308 F F F <--- 29 580-599 178 178 178 311 311 311 F F F Max gen: 180, Max flush: 315 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 90 Verifying data now No change after verify Loop: 90 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 293 293 293 F F F 1 020-039 180 180 180 314 314 314 F F F 2 040-059 180 180 180 314 314 314 F F F 3 060-079 178 178 178 311 311 311 F F F 4 080-099 178 178 178 311 311 311 F F F 5 100-119 178 178 178 311 311 311 F F F 6 120-139 178 178 178 311 311 311 F F F 7 140-159 180 180 180 314 314 314 F F F 8 160-179 180 180 180 315 315 315 F F F 9 180-199 180 180 180 314 314 314 F F F 10 200-219 179 179 179 312 312 312 F F F 11 220-239 181 181 181 316 316 316 F F F 12 240-259 176 176 176 306 306 306 F F F 13 260-279 180 180 180 314 314 314 F F F 14 280-299 180 180 180 314 314 314 F F F 15 300-319 180 180 180 314 314 314 F F F 16 320-339 181 181 181 316 316 316 F F F 17 340-359 181 181 181 316 316 316 F F F 18 360-379 180 180 180 314 314 314 F F F 19 380-399 173 173 173 301 301 301 F F F 20 400-419 178 178 178 311 311 311 F F F 21 420-439 178 178 178 311 311 311 F F F 22 440-459 178 178 178 311 311 311 F F F 23 460-479 178 178 178 311 311 311 F F F 24 480-499 180 180 180 314 314 314 F F F 25 500-519 180 180 180 314 314 314 F F F 26 520-539 178 178 178 311 311 311 F F F 27 540-559 180 180 180 314 314 314 F F F 28 560-579 180 180 180 314 314 314 F F F 29 580-599 178 178 178 311 311 311 F F F Max gen: 181, Max flush: 316 Begin loop 91 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:54.677889625Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1864,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:54.686140118Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1864,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:54.686447482Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1864} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:54.686473178Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1864} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:54.686481747Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1864} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:54.686493165Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1864} {"msg":"Crucible 311b447b-4dfc-43ce-91ed-3e0018b1d128 has session id: db74bc4b-a6d2-448c-9a67-7702a466570f","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:54.686808849Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1864,"session_id":"db74bc4b-a6d2-448c-9a67-7702a466570f"} {"msg":"Upstairs opts: Upstairs UUID: 311b447b-4dfc-43ce-91ed-3e0018b1d128, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:54.686831386Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1864,"session_id":"db74bc4b-a6d2-448c-9a67-7702a466570f"} {"msg":"Crucible stats registered with UUID: 311b447b-4dfc-43ce-91ed-3e0018b1d128","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:54.686841464Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1864,"session_id":"db74bc4b-a6d2-448c-9a67-7702a466570f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:59:54.686850623Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1864,"session_id":"db74bc4b-a6d2-448c-9a67-7702a466570f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:54.687014029Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1864,"":"io task","client":"0","":"downstairs","session_id":"db74bc4b-a6d2-448c-9a67-7702a466570f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:54.687046204Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1864,"":"io task","client":"2","":"downstairs","session_id":"db74bc4b-a6d2-448c-9a67-7702a466570f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:54.687057402Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1864,"":"io task","client":"1","":"downstairs","session_id":"db74bc4b-a6d2-448c-9a67-7702a466570f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:54.687211229Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1864,"task":"crutest"} {"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:54.687230487Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1864} {"msg":"311b447b-4dfc-43ce-91ed-3e0018b1d128 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:54.687254063Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1864,"session_id":"db74bc4b-a6d2-448c-9a67-7702a466570f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:54.687263512Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1864,"client":"0","":"downstairs","session_id":"db74bc4b-a6d2-448c-9a67-7702a466570f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:54.68727317Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1864,"client":"1","":"downstairs","session_id":"db74bc4b-a6d2-448c-9a67-7702a466570f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:54.687283189Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1864,"client":"2","":"downstairs","session_id":"db74bc4b-a6d2-448c-9a67-7702a466570f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:54.687292917Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1864,"":"io task","client":"2","":"downstairs","session_id":"db74bc4b-a6d2-448c-9a67-7702a466570f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:54.687302866Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1864,"":"io task","client":"0","":"downstairs","session_id":"db74bc4b-a6d2-448c-9a67-7702a466570f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:54.687313004Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1864,"":"io task","client":"1","":"downstairs","session_id":"db74bc4b-a6d2-448c-9a67-7702a466570f"} {"msg":"ds_connection connected from Ok(127.0.0.1:44911)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:54.687369856Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1864,"":"io task","client":"0","":"downstairs","session_id":"db74bc4b-a6d2-448c-9a67-7702a466570f"} {"msg":"ds_connection connected from Ok(127.0.0.1:39452)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:54.687395492Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1864,"":"io task","client":"2","":"downstairs","session_id":"db74bc4b-a6d2-448c-9a67-7702a466570f"} {"msg":"ds_connection connected from Ok(127.0.0.1:45841)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:54.687516664Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1864,"":"io task","client":"1","":"downstairs","session_id":"db74bc4b-a6d2-448c-9a67-7702a466570f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:54.688034108Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1864,"client":"2","":"downstairs","session_id":"db74bc4b-a6d2-448c-9a67-7702a466570f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:54.688125755Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1864,"client":"0","":"downstairs","session_id":"db74bc4b-a6d2-448c-9a67-7702a466570f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:54.688141582Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1864,"client":"1","":"downstairs","session_id":"db74bc4b-a6d2-448c-9a67-7702a466570f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:54.688342523Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1864,"session_id":"db74bc4b-a6d2-448c-9a67-7702a466570f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:54.688394705Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1864,"session_id":"db74bc4b-a6d2-448c-9a67-7702a466570f"} {"msg":"[0]R flush_numbers[0..12]: [293, 314, 314, 311, 311, 311, 311, 314, 315, 314, 312, 316]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:54.688440908Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1864,"":"downstairs","":"downstairs","session_id":"db74bc4b-a6d2-448c-9a67-7702a466570f"} {"msg":"[0]R generation[0..12]: [167, 180, 180, 178, 178, 178, 178, 180, 180, 180, 179, 181]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:54.688457926Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1864,"":"downstairs","":"downstairs","session_id":"db74bc4b-a6d2-448c-9a67-7702a466570f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:54.688469434Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1864,"":"downstairs","":"downstairs","session_id":"db74bc4b-a6d2-448c-9a67-7702a466570f"} {"msg":"[1]R flush_numbers[0..12]: [293, 314, 314, 311, 311, 311, 311, 314, 315, 314, 312, 316]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:54.688480252Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1864,"":"downstairs","":"downstairs","session_id":"db74bc4b-a6d2-448c-9a67-7702a466570f"} {"msg":"[1]R generation[0..12]: [167, 180, 180, 178, 178, 178, 178, 180, 180, 180, 179, 181]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:54.688490891Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1864,"":"downstairs","":"downstairs","session_id":"db74bc4b-a6d2-448c-9a67-7702a466570f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:54.688501849Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1864,"":"downstairs","":"downstairs","session_id":"db74bc4b-a6d2-448c-9a67-7702a466570f"} {"msg":"[2]R flush_numbers[0..12]: [293, 314, 314, 311, 311, 311, 311, 314, 315, 314, 312, 316]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:54.688512438Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1864,"":"downstairs","":"downstairs","session_id":"db74bc4b-a6d2-448c-9a67-7702a466570f"} {"msg":"[2]R generation[0..12]: [167, 180, 180, 178, 178, 178, 178, 180, 180, 180, 179, 181]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:54.688522916Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1864,"":"downstairs","":"downstairs","session_id":"db74bc4b-a6d2-448c-9a67-7702a466570f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:54.688533724Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1864,"":"downstairs","":"downstairs","session_id":"db74bc4b-a6d2-448c-9a67-7702a466570f"} {"msg":"Max found gen is 182","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:54.688543243Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1864,"":"downstairs","":"downstairs","session_id":"db74bc4b-a6d2-448c-9a67-7702a466570f"} {"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:54.688555611Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1864,"":"downstairs","":"downstairs","session_id":"db74bc4b-a6d2-448c-9a67-7702a466570f"} {"msg":"Next flush: 317","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:54.68856496Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1864,"":"downstairs","":"downstairs","session_id":"db74bc4b-a6d2-448c-9a67-7702a466570f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:54.688574688Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1864,"":"downstairs","":"downstairs","session_id":"db74bc4b-a6d2-448c-9a67-7702a466570f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:54.688584447Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1864,"session_id":"db74bc4b-a6d2-448c-9a67-7702a466570f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:54.688593306Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1864,"client":"0","":"downstairs","session_id":"db74bc4b-a6d2-448c-9a67-7702a466570f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:54.688603094Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1864,"client":"1","":"downstairs","session_id":"db74bc4b-a6d2-448c-9a67-7702a466570f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:54.688612553Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1864,"client":"2","":"downstairs","session_id":"db74bc4b-a6d2-448c-9a67-7702a466570f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:54.688622481Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1864,"session_id":"db74bc4b-a6d2-448c-9a67-7702a466570f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:54.68863166Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1864,"session_id":"db74bc4b-a6d2-448c-9a67-7702a466570f"} {"msg":"311b447b-4dfc-43ce-91ed-3e0018b1d128 is now active with session: db74bc4b-a6d2-448c-9a67-7702a466570f","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:54.688641488Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1864,"session_id":"db74bc4b-a6d2-448c-9a67-7702a466570f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:54.688650407Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1864,"session_id":"db74bc4b-a6d2-448c-9a67-7702a466570f"} {"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:54.688659456Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1864} {"msg":"Activated sub_volume 311b447b-4dfc-43ce-91ed-3e0018b1d128","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:54.688667485Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1864} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:54.688675483Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1864,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:54.689007604Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1864,"task":"crutest"} 01/30 Write block 460 len 8192 data: 11 11 02/30 Read block 305 len 28672 03/30 Read block 574 len 16384 04/30 Read block 512 len 24576 05/30 Write block 233 len 36864 data: 11 10 15 14 16 15 13 13 12 06/30 Read block 525 len 40960 07/30 Read block 471 len 24576 08/30 Read block 266 len 12288 09/30 Flush 10/30 Write block 459 len 24576 data: 13 12 12 11 11 13 11/30 Read block 43 len 12288 12/30 Read block 575 len 36864 13/30 Write block 596 len 16384 data: 13 10 6 3 14/30 Read block 266 len 12288 15/30 Read block 523 len 12288 16/30 Flush 17/30 Flush 18/30 Write block 497 len 28672 data: 11 11 14 14 13 15 17 19/30 Write block 503 len 36864 data: 18 15 14 10 10 8 7 13 12 20/30 Read block 399 len 12288 21/30 Read block 54 len 12288 22/30 Write block 61 len 20480 data: 10 11 12 12 11 23/30 Read block 575 len 20480 24/30 Write block 242 len 20480 data: 10 10 12 12 11 25/30 Read block 25 len 8192 26/30 Read block 79 len 12288 27/30 Read block 193 len 20480 28/30 Read block 396 len 36864 29/30 Read block 342 len 28672 30/30 Write block 593 len 4096 data: 17 ---------------------------------------------------------------- Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Sent Done false 1001 Acked Read 7 Done Sent Done false 1002 Acked Read 4 Done Sent Done false 1003 Acked Read 6 Done Sent Done false 1004 Acked Write 9 Done Sent Done false 1005 Acked Read 10 Done Sent Done false 1006 Acked Read 6 Done Sent Done false 1007 Acked Read 3 Done Sent Done false 1008 Acked Flush 0 Done Sent Done false 1009 Acked Write 6 Done Sent Done false 1010 Acked Read 3 Done Sent Done false 1011 Acked Read 9 Done Sent Done false 1012 Acked Write 4 Done Sent Done false 1013 Acked Read 3 Done Sent Done false 1014 Acked Read 3 Done Sent Done false 1015 Acked Flush 0 Done Sent Done false 1016 Acked Flush 0 Done Sent Done false 1017 Acked Write 7 Done Sent Done false 1018 Acked Write 9 Done Sent Done false 1019 Acked Read 3 Done Sent Done false 1020 Acked Read 3 Done Sent Done false 1021 Acked Write 5 Done Sent Done false 1022 Acked Read 5 Done Sent Done false 1023 Acked Write 5 Done Sent Done false 1024 Acked Read 2 Done Sent Done false 1025 Acked Read 3 Done Sent Done false 1026 Acked Read 5 Done Sent Done false 1027 Acked Read 9 Done Sent Done false 1028 Acked Read 7 Done Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 30 2 33 Done 29 0 28 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) None Some(JobId(1016)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:59:54.697623793Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1864} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:54.697814295Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1864,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 293 293 293 F F F 1 020-039 180 180 180 314 314 314 F F F 2 040-059 180 180 180 314 314 314 F F F 3 060-079 178 178 178 311 311 311 T F T <--- 4 080-099 178 178 178 311 311 311 F F F 5 100-119 178 178 178 311 311 311 F F F 6 120-139 178 178 178 311 311 311 F F F 7 140-159 180 180 180 314 314 314 F F F 8 160-179 180 180 180 315 315 315 F F F 9 180-199 180 180 180 314 314 314 F F F 10 200-219 179 179 179 312 312 312 F F F 11 220-239 182 181 182 317 316 317 F F F <--- 12 240-259 182 176 182 317 306 317 T F T <--- 13 260-279 180 180 180 314 314 314 F F F 14 280-299 180 180 180 314 314 314 F F F 15 300-319 180 180 180 314 314 314 F F F 16 320-339 181 181 181 316 316 316 F F F 17 340-359 181 181 181 316 316 316 F F F 18 360-379 180 180 180 314 314 314 F F F 19 380-399 173 173 173 301 301 301 F F F 20 400-419 178 178 178 311 311 311 F F F 21 420-439 178 178 178 311 311 311 F F F 22 440-459 182 178 182 318 311 318 F F F <--- 23 460-479 182 178 182 318 311 318 F F F <--- 24 480-499 180 180 180 314 314 314 T F T <--- 25 500-519 180 180 180 314 314 314 T F T <--- 26 520-539 178 178 178 311 311 311 F F F 27 540-559 180 180 180 314 314 314 F F F 28 560-579 180 180 180 314 314 314 F F F 29 580-599 182 178 182 318 311 318 T F T <--- Max gen: 182, Max flush: 318 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 91 Verifying data now No change after verify Loop: 91 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 293 293 293 F F F 1 020-039 180 180 180 314 314 314 F F F 2 040-059 180 180 180 314 314 314 F F F 3 060-079 183 183 183 319 319 319 F F F 4 080-099 178 178 178 311 311 311 F F F 5 100-119 178 178 178 311 311 311 F F F 6 120-139 178 178 178 311 311 311 F F F 7 140-159 180 180 180 314 314 314 F F F 8 160-179 180 180 180 315 315 315 F F F 9 180-199 180 180 180 314 314 314 F F F 10 200-219 179 179 179 312 312 312 F F F 11 220-239 182 182 182 317 317 317 F F F 12 240-259 183 183 183 319 319 319 F F F 13 260-279 180 180 180 314 314 314 F F F 14 280-299 180 180 180 314 314 314 F F F 15 300-319 180 180 180 314 314 314 F F F 16 320-339 181 181 181 316 316 316 F F F 17 340-359 181 181 181 316 316 316 F F F 18 360-379 180 180 180 314 314 314 F F F 19 380-399 173 173 173 301 301 301 F F F 20 400-419 178 178 178 311 311 311 F F F 21 420-439 178 178 178 311 311 311 F F F 22 440-459 182 182 182 318 318 318 F F F 23 460-479 182 182 182 318 318 318 F F F 24 480-499 183 183 183 319 319 319 F F F 25 500-519 183 183 183 319 319 319 F F F 26 520-539 178 178 178 311 311 311 F F F 27 540-559 180 180 180 314 314 314 F F F 28 560-579 180 180 180 314 314 314 F F F 29 580-599 183 183 183 319 319 319 F F F Max gen: 183, Max flush: 319 Begin loop 92 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:58.937559281Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1872,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:59:58.944640126Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1872,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:58.944695918Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1872,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:58.951726601Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1872,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:58.95200317Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1872} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:58.952027466Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1872} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:58.952036065Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1872} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:58.952047474Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1872} {"msg":"Crucible ab0025b0-ebbf-4cc7-96e8-b495dea3cd47 has session id: 5a9b9e99-c99a-4633-9a07-bd210ec95859","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:58.952416919Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1872,"session_id":"5a9b9e99-c99a-4633-9a07-bd210ec95859"} {"msg":"Upstairs opts: Upstairs UUID: ab0025b0-ebbf-4cc7-96e8-b495dea3cd47, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:58.952440096Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1872,"session_id":"5a9b9e99-c99a-4633-9a07-bd210ec95859"} {"msg":"Crucible stats registered with UUID: ab0025b0-ebbf-4cc7-96e8-b495dea3cd47","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:58.952449874Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1872,"session_id":"5a9b9e99-c99a-4633-9a07-bd210ec95859"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:59:58.952458913Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1872,"session_id":"5a9b9e99-c99a-4633-9a07-bd210ec95859"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:58.952605141Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1872,"":"io task","client":"2","":"downstairs","session_id":"5a9b9e99-c99a-4633-9a07-bd210ec95859"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:58.952638066Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1872,"":"io task","client":"0","":"downstairs","session_id":"5a9b9e99-c99a-4633-9a07-bd210ec95859"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:58.952649415Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1872,"":"io task","client":"1","":"downstairs","session_id":"5a9b9e99-c99a-4633-9a07-bd210ec95859"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:58.952766857Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1872,"task":"crutest"} {"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:58.952781335Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1872} {"msg":"ab0025b0-ebbf-4cc7-96e8-b495dea3cd47 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:58.952795913Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1872,"session_id":"5a9b9e99-c99a-4633-9a07-bd210ec95859"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:58.952805122Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1872,"client":"0","":"downstairs","session_id":"5a9b9e99-c99a-4633-9a07-bd210ec95859"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:58.95281503Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1872,"client":"1","":"downstairs","session_id":"5a9b9e99-c99a-4633-9a07-bd210ec95859"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:58.952824979Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1872,"client":"2","":"downstairs","session_id":"5a9b9e99-c99a-4633-9a07-bd210ec95859"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:58.952834697Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1872,"":"io task","client":"2","":"downstairs","session_id":"5a9b9e99-c99a-4633-9a07-bd210ec95859"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:58.952844896Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1872,"":"io task","client":"0","":"downstairs","session_id":"5a9b9e99-c99a-4633-9a07-bd210ec95859"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:58.952854824Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1872,"":"io task","client":"1","":"downstairs","session_id":"5a9b9e99-c99a-4633-9a07-bd210ec95859"} {"msg":"ds_connection connected from Ok(127.0.0.1:40572)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:58.952908576Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1872,"":"io task","client":"1","":"downstairs","session_id":"5a9b9e99-c99a-4633-9a07-bd210ec95859"} {"msg":"ds_connection connected from Ok(127.0.0.1:40124)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:58.952919825Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1872,"":"io task","client":"2","":"downstairs","session_id":"5a9b9e99-c99a-4633-9a07-bd210ec95859"} {"msg":"ds_connection connected from Ok(127.0.0.1:36699)","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:58.953072852Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1872,"":"io task","client":"0","":"downstairs","session_id":"5a9b9e99-c99a-4633-9a07-bd210ec95859"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:58.953577058Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1872,"client":"2","":"downstairs","session_id":"5a9b9e99-c99a-4633-9a07-bd210ec95859"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:58.953595425Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1872,"client":"1","":"downstairs","session_id":"5a9b9e99-c99a-4633-9a07-bd210ec95859"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:58.953685972Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1872,"client":"0","":"downstairs","session_id":"5a9b9e99-c99a-4633-9a07-bd210ec95859"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:58.95390044Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1872,"session_id":"5a9b9e99-c99a-4633-9a07-bd210ec95859"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:58.953924227Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1872,"session_id":"5a9b9e99-c99a-4633-9a07-bd210ec95859"} {"msg":"[0]R flush_numbers[0..12]: [293, 314, 314, 319, 311, 311, 311, 314, 315, 314, 312, 317]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:58.953972559Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1872,"":"downstairs","":"downstairs","session_id":"5a9b9e99-c99a-4633-9a07-bd210ec95859"} {"msg":"[0]R generation[0..12]: [167, 180, 180, 183, 178, 178, 178, 180, 180, 180, 179, 182]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:58.953984658Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1872,"":"downstairs","":"downstairs","session_id":"5a9b9e99-c99a-4633-9a07-bd210ec95859"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:58.953995726Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1872,"":"downstairs","":"downstairs","session_id":"5a9b9e99-c99a-4633-9a07-bd210ec95859"} {"msg":"[1]R flush_numbers[0..12]: [293, 314, 314, 319, 311, 311, 311, 314, 315, 314, 312, 317]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:58.954006554Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1872,"":"downstairs","":"downstairs","session_id":"5a9b9e99-c99a-4633-9a07-bd210ec95859"} {"msg":"[1]R generation[0..12]: [167, 180, 180, 183, 178, 178, 178, 180, 180, 180, 179, 182]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:58.954017383Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1872,"":"downstairs","":"downstairs","session_id":"5a9b9e99-c99a-4633-9a07-bd210ec95859"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:58.954028381Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1872,"":"downstairs","":"downstairs","session_id":"5a9b9e99-c99a-4633-9a07-bd210ec95859"} {"msg":"[2]R flush_numbers[0..12]: [293, 314, 314, 319, 311, 311, 311, 314, 315, 314, 312, 317]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:58.95403907Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1872,"":"downstairs","":"downstairs","session_id":"5a9b9e99-c99a-4633-9a07-bd210ec95859"} {"msg":"[2]R generation[0..12]: [167, 180, 180, 183, 178, 178, 178, 180, 180, 180, 179, 182]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:58.954049838Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1872,"":"downstairs","":"downstairs","session_id":"5a9b9e99-c99a-4633-9a07-bd210ec95859"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:58.954060776Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1872,"":"downstairs","":"downstairs","session_id":"5a9b9e99-c99a-4633-9a07-bd210ec95859"} {"msg":"Max found gen is 184","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:58.954070265Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1872,"":"downstairs","":"downstairs","session_id":"5a9b9e99-c99a-4633-9a07-bd210ec95859"} {"msg":"Generation requested: 184 >= found:184","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:58.954080004Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1872,"":"downstairs","":"downstairs","session_id":"5a9b9e99-c99a-4633-9a07-bd210ec95859"} {"msg":"Next flush: 320","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:58.954089562Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1872,"":"downstairs","":"downstairs","session_id":"5a9b9e99-c99a-4633-9a07-bd210ec95859"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:58.954098931Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1872,"":"downstairs","":"downstairs","session_id":"5a9b9e99-c99a-4633-9a07-bd210ec95859"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:58.954117328Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1872,"session_id":"5a9b9e99-c99a-4633-9a07-bd210ec95859"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:58.954126307Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1872,"client":"0","":"downstairs","session_id":"5a9b9e99-c99a-4633-9a07-bd210ec95859"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:58.954137855Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1872,"client":"1","":"downstairs","session_id":"5a9b9e99-c99a-4633-9a07-bd210ec95859"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:58.954147604Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1872,"client":"2","":"downstairs","session_id":"5a9b9e99-c99a-4633-9a07-bd210ec95859"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:58.954157302Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1872,"session_id":"5a9b9e99-c99a-4633-9a07-bd210ec95859"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:58.954166371Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1872,"session_id":"5a9b9e99-c99a-4633-9a07-bd210ec95859"} {"msg":"ab0025b0-ebbf-4cc7-96e8-b495dea3cd47 is now active with session: 5a9b9e99-c99a-4633-9a07-bd210ec95859","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:58.954176379Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1872,"session_id":"5a9b9e99-c99a-4633-9a07-bd210ec95859"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:58.954185118Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1872,"session_id":"5a9b9e99-c99a-4633-9a07-bd210ec95859"} {"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:58.954194227Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1872} {"msg":"Activated sub_volume ab0025b0-ebbf-4cc7-96e8-b495dea3cd47","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:58.954202256Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1872} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:58.954210254Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1872,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:58.954479055Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1872,"task":"crutest"} 01/30 Write block 147 len 16384 data: 18 15 15 14 02/30 Write block 216 len 32768 data: 10 7 6 6 9 14 15 14 03/30 Write block 542 len 28672 data: 13 11 12 13 15 15 15 04/30 Read block 1 len 8192 05/30 Read block 206 len 36864 06/30 Flush 07/30 Flush 08/30 Write block 448 len 40960 data: 22 20 21 22 19 20 18 14 15 18 09/30 Write block 356 len 20480 data: 13 15 14 12 14 10/30 Read block 456 len 40960 11/30 Flush 12/30 Write block 445 len 36864 data: 17 17 18 23 21 22 23 20 21 13/30 Write block 515 len 36864 data: 17 17 17 18 16 15 15 12 8 14/30 Write block 25 len 8192 data: 17 18 15/30 Write block 414 len 36864 data: 16 18 19 14 12 12 11 11 13 16/30 Read block 426 len 40960 17/30 Read block 252 len 40960 18/30 Read block 78 len 28672 19/30 Write block 364 len 32768 data: 14 17 16 14 16 15 16 14 20/30 Write block 523 len 8192 data: 9 10 21/30 Read block 343 len 16384 22/30 Flush 23/30 Write block 261 len 8192 data: 17 15 24/30 Write block 288 len 4096 data: 17 25/30 Write block 101 len 8192 data: 19 13 26/30 Read block 102 len 12288 27/30 Read block 76 len 20480 28/30 Write block 74 len 16384 data: 11 11 10 9 29/30 Write block 184 len 8192 data: 13 10 30/30 Read block 426 len 16384 ---------------------------------------------------------------- Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Flush 0 Sent Done Done false 1007 Acked Write 10 Sent Done Done false 1008 Acked Write 5 Sent Done Done false 1009 Acked Read 10 Sent Done Done false 1010 Acked Flush 0 Sent Done Done false 1011 Acked Write 9 Sent Done Done false 1012 Acked Write 9 Sent Done Done false 1013 Acked Write 2 Sent Done Done false 1014 Acked Write 9 Sent Done Done false 1015 Acked Read 10 Sent Done Done false 1016 Acked Read 10 Sent Done Done false 1017 Acked Read 7 Sent Done Done false 1018 Acked Write 8 Sent Done Done false 1019 Acked Write 2 Sent Done Done false 1020 Acked Read 4 Sent Done Done false 1021 Acked Flush 0 Sent Done Done false 1022 Acked Write 2 Sent Done Done false 1023 Acked Write 1 Sent Done Done false 1024 Acked Write 2 Sent Done Done false 1025 Acked Read 3 Sent Done Done false 1026 Acked Read 5 Sent Done Done false 1027 Acked Write 4 Sent Done Done false 1028 Acked Write 2 Sent Done Sent false 1029 Acked Read 4 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 24 0 2 26 Done 0 24 22 46 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1005)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T18:59:58.963130388Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1872} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T18:59:58.96332326Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1872,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 293 293 293 F F F 1 020-039 180 184 184 314 323 323 F F F <--- 2 040-059 180 180 180 314 314 314 F F F 3 060-079 183 183 183 319 319 319 F T T <--- 4 080-099 178 178 178 311 311 311 F F F 5 100-119 178 178 178 311 311 311 F T T <--- 6 120-139 178 178 178 311 311 311 F F F 7 140-159 184 184 184 320 320 320 F F F 8 160-179 180 180 180 315 315 315 F F F 9 180-199 180 180 180 314 314 314 F T T <--- 10 200-219 184 184 184 320 320 320 F F F 11 220-239 184 184 184 320 320 320 F F F 12 240-259 183 183 183 319 319 319 F F F 13 260-279 180 180 180 314 314 314 F T T <--- 14 280-299 180 180 180 314 314 314 F T T <--- 15 300-319 180 180 180 314 314 314 F F F 16 320-339 181 181 181 316 316 316 F F F 17 340-359 181 184 184 316 322 322 F F F <--- 18 360-379 180 184 184 314 323 323 F F F <--- 19 380-399 173 173 173 301 301 301 F F F 20 400-419 178 184 184 311 323 323 F F F <--- 21 420-439 178 184 184 311 323 323 F F F <--- 22 440-459 182 184 184 318 323 323 F F F <--- 23 460-479 182 182 182 318 318 318 F F F 24 480-499 183 183 183 319 319 319 F F F 25 500-519 183 184 184 319 323 323 F F F <--- 26 520-539 178 184 184 311 323 323 F F F <--- 27 540-559 184 184 184 320 320 320 F F F 28 560-579 180 180 180 314 314 314 F F F 29 580-599 183 183 183 319 319 319 F F F Max gen: 184, Max flush: 323 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 92 Verifying data now No change after verify Loop: 92 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 293 293 293 F F F 1 020-039 184 184 184 323 323 323 F F F 2 040-059 180 180 180 314 314 314 F F F 3 060-079 185 185 185 324 324 324 F F F 4 080-099 178 178 178 311 311 311 F F F 5 100-119 185 185 185 324 324 324 F F F 6 120-139 178 178 178 311 311 311 F F F 7 140-159 184 184 184 320 320 320 F F F 8 160-179 180 180 180 315 315 315 F F F 9 180-199 185 185 185 324 324 324 F F F 10 200-219 184 184 184 320 320 320 F F F 11 220-239 184 184 184 320 320 320 F F F 12 240-259 183 183 183 319 319 319 F F F 13 260-279 185 185 185 324 324 324 F F F 14 280-299 185 185 185 324 324 324 F F F 15 300-319 180 180 180 314 314 314 F F F 16 320-339 181 181 181 316 316 316 F F F 17 340-359 184 184 184 322 322 322 F F F 18 360-379 184 184 184 323 323 323 F F F 19 380-399 173 173 173 301 301 301 F F F 20 400-419 184 184 184 323 323 323 F F F 21 420-439 184 184 184 323 323 323 F F F 22 440-459 184 184 184 323 323 323 F F F 23 460-479 182 182 182 318 318 318 F F F 24 480-499 183 183 183 319 319 319 F F F 25 500-519 184 184 184 323 323 323 F F F 26 520-539 184 184 184 323 323 323 F F F 27 540-559 184 184 184 320 320 320 F F F 28 560-579 180 180 180 314 314 314 F F F 29 580-599 183 183 183 319 319 319 F F F Max gen: 185, Max flush: 324 Begin loop 93 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:03.220457541Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1880,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T19:00:03.227571941Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1880,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:03.227621424Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1880,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:03.234394685Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1880,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:03.234684702Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1880} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:03.234712308Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1880} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:03.234724536Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1880} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:03.234736344Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1880} {"msg":"Crucible 23387552-63fa-4430-ae0d-d86ec63dd8d0 has session id: 4b357cbf-f3eb-43d0-b119-b819cb153cb5","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:03.235108269Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1880,"session_id":"4b357cbf-f3eb-43d0-b119-b819cb153cb5"} {"msg":"Upstairs opts: Upstairs UUID: 23387552-63fa-4430-ae0d-d86ec63dd8d0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:03.235137815Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1880,"session_id":"4b357cbf-f3eb-43d0-b119-b819cb153cb5"} {"msg":"Crucible stats registered with UUID: 23387552-63fa-4430-ae0d-d86ec63dd8d0","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:03.235148983Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1880,"session_id":"4b357cbf-f3eb-43d0-b119-b819cb153cb5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T19:00:03.235158312Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1880,"session_id":"4b357cbf-f3eb-43d0-b119-b819cb153cb5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:03.235290183Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1880,"":"io task","client":"0","":"downstairs","session_id":"4b357cbf-f3eb-43d0-b119-b819cb153cb5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:03.23530896Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1880,"":"io task","client":"1","":"downstairs","session_id":"4b357cbf-f3eb-43d0-b119-b819cb153cb5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:03.235319568Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1880,"":"io task","client":"2","":"downstairs","session_id":"4b357cbf-f3eb-43d0-b119-b819cb153cb5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:03.235455608Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1880,"task":"crutest"} {"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:03.235474545Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1880} {"msg":"23387552-63fa-4430-ae0d-d86ec63dd8d0 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:03.235498822Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1880,"session_id":"4b357cbf-f3eb-43d0-b119-b819cb153cb5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:03.23550835Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1880,"client":"0","":"downstairs","session_id":"4b357cbf-f3eb-43d0-b119-b819cb153cb5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:03.235518249Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1880,"client":"1","":"downstairs","session_id":"4b357cbf-f3eb-43d0-b119-b819cb153cb5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:03.235528038Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1880,"client":"2","":"downstairs","session_id":"4b357cbf-f3eb-43d0-b119-b819cb153cb5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:03.235537646Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1880,"":"io task","client":"2","":"downstairs","session_id":"4b357cbf-f3eb-43d0-b119-b819cb153cb5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:03.235549544Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1880,"":"io task","client":"0","":"downstairs","session_id":"4b357cbf-f3eb-43d0-b119-b819cb153cb5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:03.235559653Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1880,"":"io task","client":"1","":"downstairs","session_id":"4b357cbf-f3eb-43d0-b119-b819cb153cb5"} {"msg":"ds_connection connected from Ok(127.0.0.1:55124)","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:03.235658798Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1880,"":"io task","client":"1","":"downstairs","session_id":"4b357cbf-f3eb-43d0-b119-b819cb153cb5"} {"msg":"ds_connection connected from Ok(127.0.0.1:61631)","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:03.235684794Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1880,"":"io task","client":"2","":"downstairs","session_id":"4b357cbf-f3eb-43d0-b119-b819cb153cb5"} {"msg":"ds_connection connected from Ok(127.0.0.1:44966)","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:03.235816465Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1880,"":"io task","client":"0","":"downstairs","session_id":"4b357cbf-f3eb-43d0-b119-b819cb153cb5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:03.23632567Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1880,"client":"2","":"downstairs","session_id":"4b357cbf-f3eb-43d0-b119-b819cb153cb5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:03.236368854Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1880,"client":"1","":"downstairs","session_id":"4b357cbf-f3eb-43d0-b119-b819cb153cb5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:03.236383231Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1880,"client":"0","":"downstairs","session_id":"4b357cbf-f3eb-43d0-b119-b819cb153cb5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:03.23666305Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1880,"session_id":"4b357cbf-f3eb-43d0-b119-b819cb153cb5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:03.236699475Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1880,"session_id":"4b357cbf-f3eb-43d0-b119-b819cb153cb5"} {"msg":"[0]R flush_numbers[0..12]: [293, 323, 314, 324, 311, 324, 311, 320, 315, 324, 320, 320]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:03.236793231Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1880,"":"downstairs","":"downstairs","session_id":"4b357cbf-f3eb-43d0-b119-b819cb153cb5"} {"msg":"[0]R generation[0..12]: [167, 184, 180, 185, 178, 185, 178, 184, 180, 185, 184, 184]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:03.236804949Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1880,"":"downstairs","":"downstairs","session_id":"4b357cbf-f3eb-43d0-b119-b819cb153cb5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:03.236816317Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1880,"":"downstairs","":"downstairs","session_id":"4b357cbf-f3eb-43d0-b119-b819cb153cb5"} {"msg":"[1]R flush_numbers[0..12]: [293, 323, 314, 324, 311, 324, 311, 320, 315, 324, 320, 320]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:03.236827166Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1880,"":"downstairs","":"downstairs","session_id":"4b357cbf-f3eb-43d0-b119-b819cb153cb5"} {"msg":"[1]R generation[0..12]: [167, 184, 180, 185, 178, 185, 178, 184, 180, 185, 184, 184]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:03.236839974Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1880,"":"downstairs","":"downstairs","session_id":"4b357cbf-f3eb-43d0-b119-b819cb153cb5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:03.236851092Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1880,"":"downstairs","":"downstairs","session_id":"4b357cbf-f3eb-43d0-b119-b819cb153cb5"} {"msg":"[2]R flush_numbers[0..12]: [293, 323, 314, 324, 311, 324, 311, 320, 315, 324, 320, 320]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:03.236861871Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1880,"":"downstairs","":"downstairs","session_id":"4b357cbf-f3eb-43d0-b119-b819cb153cb5"} {"msg":"[2]R generation[0..12]: [167, 184, 180, 185, 178, 185, 178, 184, 180, 185, 184, 184]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:03.236872659Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1880,"":"downstairs","":"downstairs","session_id":"4b357cbf-f3eb-43d0-b119-b819cb153cb5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:03.236883888Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1880,"":"downstairs","":"downstairs","session_id":"4b357cbf-f3eb-43d0-b119-b819cb153cb5"} {"msg":"Max found gen is 186","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:03.236893596Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1880,"":"downstairs","":"downstairs","session_id":"4b357cbf-f3eb-43d0-b119-b819cb153cb5"} {"msg":"Generation requested: 186 >= found:186","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:03.236903585Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1880,"":"downstairs","":"downstairs","session_id":"4b357cbf-f3eb-43d0-b119-b819cb153cb5"} {"msg":"Next flush: 325","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:03.236912983Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1880,"":"downstairs","":"downstairs","session_id":"4b357cbf-f3eb-43d0-b119-b819cb153cb5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:03.236922472Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1880,"":"downstairs","":"downstairs","session_id":"4b357cbf-f3eb-43d0-b119-b819cb153cb5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:03.23693209Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1880,"session_id":"4b357cbf-f3eb-43d0-b119-b819cb153cb5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:03.236940879Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1880,"client":"0","":"downstairs","session_id":"4b357cbf-f3eb-43d0-b119-b819cb153cb5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:03.236950878Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1880,"client":"1","":"downstairs","session_id":"4b357cbf-f3eb-43d0-b119-b819cb153cb5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:03.236960886Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1880,"client":"2","":"downstairs","session_id":"4b357cbf-f3eb-43d0-b119-b819cb153cb5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:03.236970865Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1880,"session_id":"4b357cbf-f3eb-43d0-b119-b819cb153cb5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:03.236980053Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1880,"session_id":"4b357cbf-f3eb-43d0-b119-b819cb153cb5"} {"msg":"23387552-63fa-4430-ae0d-d86ec63dd8d0 is now active with session: 4b357cbf-f3eb-43d0-b119-b819cb153cb5","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:03.236991132Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1880,"session_id":"4b357cbf-f3eb-43d0-b119-b819cb153cb5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:03.237011389Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1880,"session_id":"4b357cbf-f3eb-43d0-b119-b819cb153cb5"} {"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:03.237020797Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1880} {"msg":"Activated sub_volume 23387552-63fa-4430-ae0d-d86ec63dd8d0","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:03.237028826Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1880} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:03.237036945Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1880,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:03.237348849Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1880,"task":"crutest"} 01/30 Write block 188 len 40960 data: 13 12 12 18 20 20 20 20 17 18 02/30 Write block 497 len 28672 data: 12 12 15 15 14 16 19 03/30 Read block 147 len 40960 04/30 Read block 376 len 32768 05/30 Read block 323 len 36864 06/30 Read block 158 len 36864 07/30 Flush 08/30 Write block 27 len 32768 data: 16 19 18 16 15 13 10 11 09/30 Read block 554 len 12288 10/30 Read block 104 len 40960 11/30 Write block 498 len 8192 data: 13 16 12/30 Read block 125 len 12288 13/30 Write block 395 len 40960 data: 13 12 13 12 12 11 8 7 12 12 14/30 Flush 15/30 Write block 278 len 8192 data: 17 13 16/30 Write block 473 len 36864 data: 11 14 16 14 16 18 19 19 17 17/30 Write block 586 len 36864 data: 16 18 15 14 15 16 17 18 16 18/30 Read block 169 len 4096 19/30 Write block 401 len 4096 data: 9 20/30 Read block 478 len 16384 21/30 Read block 224 len 16384 22/30 Read block 275 len 8192 23/30 Write block 191 len 20480 data: 19 21 21 21 21 24/30 Read block 191 len 40960 25/30 Read block 5 len 4096 26/30 Write block 371 len 28672 data: 15 16 15 14 14 13 13 27/30 Read block 8 len 4096 28/30 Read block 573 len 12288 29/30 Write block 418 len 4096 data: 13 30/30 Write block 382 len 20480 data: 17 16 13 12 12 ---------------------------------------------------------------- Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1007 Acked Write 8 Sent Done Done false 1008 Acked Read 3 Sent Done Done false 1009 Acked Read 10 Sent Done Done false 1010 Acked Write 2 Sent Done Done false 1011 Acked Read 3 Sent Done Done false 1012 Acked Write 10 Sent Done Done false 1013 Acked Flush 0 Sent Done Done false 1014 Acked Write 2 Sent Done Done false 1015 Acked Write 9 Sent Done Done false 1016 Acked Write 9 Sent Done Done false 1017 Acked Read 1 Sent Done Done false 1018 Acked Write 1 Sent Done Done false 1019 Acked Read 4 Sent Done Done false 1020 Acked Read 4 Sent Done Done false 1021 Acked Read 2 Sent Done Done false 1022 Acked Write 5 Sent Done Done false 1023 Acked Read 10 Sent Done Done false 1024 Acked Read 1 Sent Done Done false 1025 Acked Write 7 Sent Done Done false 1026 Acked Read 1 Sent Done Done false 1027 Acked Read 3 Sent Done Done false 1028 Acked Write 1 Sent Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 23 1 1 25 Done 0 22 22 44 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1006)) Some(JobId(1013)) Some(JobId(1013)) Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T19:00:03.246599574Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1880} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:03.246787546Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1880,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 293 293 293 F F F 1 020-039 184 186 186 323 326 326 F F F <--- 2 040-059 180 180 180 314 314 314 F F F 3 060-079 185 185 185 324 324 324 F F F 4 080-099 178 178 178 311 311 311 F F F 5 100-119 185 185 185 324 324 324 F F F 6 120-139 178 178 178 311 311 311 F F F 7 140-159 184 184 184 320 320 320 F F F 8 160-179 180 180 180 315 315 315 F F F 9 180-199 186 186 186 325 325 325 F T T <--- 10 200-219 184 184 184 320 320 320 F F F 11 220-239 184 184 184 320 320 320 F F F 12 240-259 183 183 183 319 319 319 F F F 13 260-279 185 185 185 324 324 324 F T T <--- 14 280-299 185 185 185 324 324 324 F F F 15 300-319 180 180 180 314 314 314 F F F 16 320-339 181 181 181 316 316 316 F F F 17 340-359 184 184 184 322 322 322 F F F 18 360-379 184 184 184 323 323 323 F T T <--- 19 380-399 173 186 186 301 326 326 F T T <--- 20 400-419 184 186 186 323 326 326 F T T <--- 21 420-439 184 184 184 323 323 323 F F F 22 440-459 184 184 184 323 323 323 F F F 23 460-479 182 182 182 318 318 318 F T T <--- 24 480-499 186 186 186 325 326 326 F T T <--- 25 500-519 186 186 186 325 325 325 F F F 26 520-539 184 184 184 323 323 323 F F F 27 540-559 184 184 184 320 320 320 F F F 28 560-579 180 180 180 314 314 314 F F F 29 580-599 183 183 183 319 319 319 F T T <--- Max gen: 186, Max flush: 326 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 93 Verifying data now No change after verify Loop: 93 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 293 293 293 F F F 1 020-039 186 186 186 326 326 326 F F F 2 040-059 180 180 180 314 314 314 F F F 3 060-079 185 185 185 324 324 324 F F F 4 080-099 178 178 178 311 311 311 F F F 5 100-119 185 185 185 324 324 324 F F F 6 120-139 178 178 178 311 311 311 F F F 7 140-159 184 184 184 320 320 320 F F F 8 160-179 180 180 180 315 315 315 F F F 9 180-199 187 187 187 327 327 327 F F F 10 200-219 184 184 184 320 320 320 F F F 11 220-239 184 184 184 320 320 320 F F F 12 240-259 183 183 183 319 319 319 F F F 13 260-279 187 187 187 327 327 327 F F F 14 280-299 185 185 185 324 324 324 F F F 15 300-319 180 180 180 314 314 314 F F F 16 320-339 181 181 181 316 316 316 F F F 17 340-359 184 184 184 322 322 322 F F F 18 360-379 187 187 187 327 327 327 F F F 19 380-399 187 187 187 327 327 327 F F F 20 400-419 187 187 187 327 327 327 F F F 21 420-439 184 184 184 323 323 323 F F F 22 440-459 184 184 184 323 323 323 F F F 23 460-479 187 187 187 327 327 327 F F F 24 480-499 187 187 187 327 327 327 F F F 25 500-519 186 186 186 325 325 325 F F F 26 520-539 184 184 184 323 323 323 F F F 27 540-559 184 184 184 320 320 320 F F F 28 560-579 180 180 180 314 314 314 F F F 29 580-599 187 187 187 327 327 327 F F F Max gen: 187, Max flush: 327 Begin loop 94 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:07.497443313Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1888,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T19:00:07.504524639Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1888,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:07.5045805Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1888,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:07.511349892Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1888,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:07.511615432Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1888} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:07.511640239Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1888} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:07.511648557Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1888} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:07.511659986Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1888} {"msg":"Crucible f427db88-caad-40fd-a82a-54e6ff664933 has session id: 1db5d87b-a370-4caf-8419-342fc15fdd7c","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:07.51203563Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1888,"session_id":"1db5d87b-a370-4caf-8419-342fc15fdd7c"} {"msg":"Upstairs opts: Upstairs UUID: f427db88-caad-40fd-a82a-54e6ff664933, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:07.512056047Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1888,"session_id":"1db5d87b-a370-4caf-8419-342fc15fdd7c"} {"msg":"Crucible stats registered with UUID: f427db88-caad-40fd-a82a-54e6ff664933","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:07.512067766Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1888,"session_id":"1db5d87b-a370-4caf-8419-342fc15fdd7c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T19:00:07.512076834Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1888,"session_id":"1db5d87b-a370-4caf-8419-342fc15fdd7c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:07.512217963Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1888,"":"io task","client":"0","":"downstairs","session_id":"1db5d87b-a370-4caf-8419-342fc15fdd7c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:07.512233891Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1888,"":"io task","client":"2","":"downstairs","session_id":"1db5d87b-a370-4caf-8419-342fc15fdd7c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:07.51224421Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1888,"":"io task","client":"1","":"downstairs","session_id":"1db5d87b-a370-4caf-8419-342fc15fdd7c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:07.512406096Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1888,"task":"crutest"} {"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:07.512416754Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1888} {"msg":"f427db88-caad-40fd-a82a-54e6ff664933 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:07.512434312Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1888,"session_id":"1db5d87b-a370-4caf-8419-342fc15fdd7c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:07.51244318Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1888,"client":"0","":"downstairs","session_id":"1db5d87b-a370-4caf-8419-342fc15fdd7c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:07.512452789Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1888,"client":"1","":"downstairs","session_id":"1db5d87b-a370-4caf-8419-342fc15fdd7c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:07.512462607Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1888,"client":"2","":"downstairs","session_id":"1db5d87b-a370-4caf-8419-342fc15fdd7c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:07.512472076Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1888,"":"io task","client":"2","":"downstairs","session_id":"1db5d87b-a370-4caf-8419-342fc15fdd7c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:07.512482134Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1888,"":"io task","client":"0","":"downstairs","session_id":"1db5d87b-a370-4caf-8419-342fc15fdd7c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:07.512492203Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1888,"":"io task","client":"1","":"downstairs","session_id":"1db5d87b-a370-4caf-8419-342fc15fdd7c"} {"msg":"ds_connection connected from Ok(127.0.0.1:59121)","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:07.512541946Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1888,"":"io task","client":"1","":"downstairs","session_id":"1db5d87b-a370-4caf-8419-342fc15fdd7c"} {"msg":"ds_connection connected from Ok(127.0.0.1:58345)","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:07.512553104Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1888,"":"io task","client":"0","":"downstairs","session_id":"1db5d87b-a370-4caf-8419-342fc15fdd7c"} {"msg":"ds_connection connected from Ok(127.0.0.1:48722)","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:07.512563392Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1888,"":"io task","client":"2","":"downstairs","session_id":"1db5d87b-a370-4caf-8419-342fc15fdd7c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:07.513227125Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1888,"client":"0","":"downstairs","session_id":"1db5d87b-a370-4caf-8419-342fc15fdd7c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:07.513239453Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1888,"client":"1","":"downstairs","session_id":"1db5d87b-a370-4caf-8419-342fc15fdd7c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:07.513252921Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1888,"client":"2","":"downstairs","session_id":"1db5d87b-a370-4caf-8419-342fc15fdd7c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:07.513590521Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1888,"session_id":"1db5d87b-a370-4caf-8419-342fc15fdd7c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:07.513601449Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1888,"session_id":"1db5d87b-a370-4caf-8419-342fc15fdd7c"} {"msg":"[0]R flush_numbers[0..12]: [293, 326, 314, 324, 311, 324, 311, 320, 315, 327, 320, 320]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:07.513620127Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1888,"":"downstairs","":"downstairs","session_id":"1db5d87b-a370-4caf-8419-342fc15fdd7c"} {"msg":"[0]R generation[0..12]: [167, 186, 180, 185, 178, 185, 178, 184, 180, 187, 184, 184]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:07.513630965Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1888,"":"downstairs","":"downstairs","session_id":"1db5d87b-a370-4caf-8419-342fc15fdd7c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:07.513642113Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1888,"":"downstairs","":"downstairs","session_id":"1db5d87b-a370-4caf-8419-342fc15fdd7c"} {"msg":"[1]R flush_numbers[0..12]: [293, 326, 314, 324, 311, 324, 311, 320, 315, 327, 320, 320]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:07.513652852Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1888,"":"downstairs","":"downstairs","session_id":"1db5d87b-a370-4caf-8419-342fc15fdd7c"} {"msg":"[1]R generation[0..12]: [167, 186, 180, 185, 178, 185, 178, 184, 180, 187, 184, 184]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:07.51366336Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1888,"":"downstairs","":"downstairs","session_id":"1db5d87b-a370-4caf-8419-342fc15fdd7c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:07.513674499Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1888,"":"downstairs","":"downstairs","session_id":"1db5d87b-a370-4caf-8419-342fc15fdd7c"} {"msg":"[2]R flush_numbers[0..12]: [293, 326, 314, 324, 311, 324, 311, 320, 315, 327, 320, 320]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:07.513685177Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1888,"":"downstairs","":"downstairs","session_id":"1db5d87b-a370-4caf-8419-342fc15fdd7c"} {"msg":"[2]R generation[0..12]: [167, 186, 180, 185, 178, 185, 178, 184, 180, 187, 184, 184]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:07.513695955Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1888,"":"downstairs","":"downstairs","session_id":"1db5d87b-a370-4caf-8419-342fc15fdd7c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:07.513708664Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1888,"":"downstairs","":"downstairs","session_id":"1db5d87b-a370-4caf-8419-342fc15fdd7c"} {"msg":"Max found gen is 188","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:07.513718152Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1888,"":"downstairs","":"downstairs","session_id":"1db5d87b-a370-4caf-8419-342fc15fdd7c"} {"msg":"Generation requested: 188 >= found:188","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:07.513728151Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1888,"":"downstairs","":"downstairs","session_id":"1db5d87b-a370-4caf-8419-342fc15fdd7c"} {"msg":"Next flush: 328","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:07.513737619Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1888,"":"downstairs","":"downstairs","session_id":"1db5d87b-a370-4caf-8419-342fc15fdd7c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:07.513747018Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1888,"":"downstairs","":"downstairs","session_id":"1db5d87b-a370-4caf-8419-342fc15fdd7c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:07.513756806Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1888,"session_id":"1db5d87b-a370-4caf-8419-342fc15fdd7c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:07.513765595Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1888,"client":"0","":"downstairs","session_id":"1db5d87b-a370-4caf-8419-342fc15fdd7c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:07.513775424Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1888,"client":"1","":"downstairs","session_id":"1db5d87b-a370-4caf-8419-342fc15fdd7c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:07.513785312Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1888,"client":"2","":"downstairs","session_id":"1db5d87b-a370-4caf-8419-342fc15fdd7c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:07.513795381Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1888,"session_id":"1db5d87b-a370-4caf-8419-342fc15fdd7c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:07.513804639Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1888,"session_id":"1db5d87b-a370-4caf-8419-342fc15fdd7c"} {"msg":"f427db88-caad-40fd-a82a-54e6ff664933 is now active with session: 1db5d87b-a370-4caf-8419-342fc15fdd7c","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:07.513814678Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1888,"session_id":"1db5d87b-a370-4caf-8419-342fc15fdd7c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:07.513823647Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1888,"session_id":"1db5d87b-a370-4caf-8419-342fc15fdd7c"} {"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:07.513832845Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1888} {"msg":"Activated sub_volume f427db88-caad-40fd-a82a-54e6ff664933","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:07.513840964Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1888} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:07.513849203Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1888,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:07.514180004Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1888,"task":"crutest"} 01/30 Write block 115 len 20480 data: 13 15 16 15 16 02/30 Write block 292 len 20480 data: 14 13 9 6 3 03/30 Flush 04/30 Write block 318 len 8192 data: 8 9 05/30 Write block 59 len 4096 data: 8 06/30 Read block 56 len 24576 07/30 Flush 08/30 Flush 09/30 Write block 373 len 16384 data: 16 15 15 14 10/30 Read block 162 len 40960 11/30 Read block 379 len 20480 12/30 Write block 207 len 24576 data: 23 21 19 16 12 14 13/30 Write block 558 len 8192 data: 13 12 14/30 Write block 305 len 12288 data: 9 11 10 15/30 Read block 474 len 28672 16/30 Write block 335 len 20480 data: 11 11 13 14 16 17/30 Write block 406 len 32768 data: 11 11 13 13 15 20 17 17 18/30 Write block 110 len 8192 data: 10 14 19/30 Write block 163 len 12288 data: 12 13 15 20/30 Write block 142 len 4096 data: 19 21/30 Write block 91 len 16384 data: 15 16 16 13 22/30 Read block 503 len 12288 23/30 Write block 180 len 24576 data: 13 12 12 12 14 11 24/30 Read block 337 len 4096 25/30 Flush 26/30 Write block 350 len 8192 data: 8 10 27/30 Read block 284 len 12288 28/30 Read block 184 len 8192 29/30 Read block 532 len 24576 30/30 Write block 59 len 24576 data: 9 11 11 12 13 13 ---------------------------------------------------------------- Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:22 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1008 Acked Write 4 Done Done Done false 1009 Acked Read 10 Done Done Done false 1010 Acked Read 5 Done Done Done false 1011 Acked Write 6 Done Done Done false 1012 Acked Write 2 Sent Done Done false 1013 Acked Write 3 Sent Done Done false 1014 Acked Read 7 Sent Done Done false 1015 Acked Write 5 Sent Done Done false 1016 Acked Write 8 Sent Done Done false 1017 Acked Write 2 Sent Done Done false 1018 Acked Write 3 Sent Done Done false 1019 Acked Write 1 Sent Done Done false 1020 Acked Write 4 Sent Done Done false 1021 Acked Read 3 Sent Done Done false 1022 Acked Write 6 Sent Done Done false 1023 Acked Read 1 Sent Done Done false 1024 Acked Flush 0 Sent Done Done false 1025 Acked Write 2 Sent Done Done false 1026 Acked Read 3 Sent Done Done false 1027 Acked Read 2 Sent Done Done false 1028 Acked Read 6 Sent Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 18 1 1 20 Done 4 21 21 46 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1007)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T19:00:07.521649292Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1888} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:07.52186443Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1888,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 293 293 293 F F F 1 020-039 186 186 186 326 326 326 F F F 2 040-059 188 188 188 329 329 329 F T T <--- 3 060-079 185 185 185 324 324 324 F T T <--- 4 080-099 178 188 188 311 331 331 F F F <--- 5 100-119 188 188 188 328 331 331 F F F <--- 6 120-139 178 178 178 311 311 311 F F F 7 140-159 184 188 188 320 331 331 F F F <--- 8 160-179 180 188 188 315 331 331 F F F <--- 9 180-199 187 188 188 327 331 331 F F F <--- 10 200-219 184 188 188 320 331 331 T F F <--- 11 220-239 184 184 184 320 320 320 F F F 12 240-259 183 183 183 319 319 319 F F F 13 260-279 187 187 187 327 327 327 F F F 14 280-299 188 188 188 328 328 328 F F F 15 300-319 188 188 188 329 331 331 F F F <--- 16 320-339 181 188 188 316 331 331 F F F <--- 17 340-359 184 184 184 322 322 322 F T T <--- 18 360-379 187 188 188 327 331 331 T F F <--- 19 380-399 187 187 187 327 327 327 F F F 20 400-419 187 188 188 327 331 331 F F F <--- 21 420-439 184 184 184 323 323 323 F F F 22 440-459 184 184 184 323 323 323 F F F 23 460-479 187 187 187 327 327 327 F F F 24 480-499 187 187 187 327 327 327 F F F 25 500-519 186 186 186 325 325 325 F F F 26 520-539 184 184 184 323 323 323 F F F 27 540-559 184 188 188 320 331 331 F F F <--- 28 560-579 180 180 180 314 314 314 F F F 29 580-599 187 187 187 327 327 327 F F F Max gen: 188, Max flush: 331 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 94 Verifying data now No change after verify Loop: 94 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 293 293 293 F F F 1 020-039 186 186 186 326 326 326 F F F 2 040-059 189 189 189 332 332 332 F F F 3 060-079 189 189 189 332 332 332 F F F 4 080-099 188 188 188 331 331 331 F F F 5 100-119 188 188 188 331 331 331 F F F 6 120-139 178 178 178 311 311 311 F F F 7 140-159 188 188 188 331 331 331 F F F 8 160-179 188 188 188 331 331 331 F F F 9 180-199 188 188 188 331 331 331 F F F 10 200-219 188 188 188 331 331 331 F F F 11 220-239 184 184 184 320 320 320 F F F 12 240-259 183 183 183 319 319 319 F F F 13 260-279 187 187 187 327 327 327 F F F 14 280-299 188 188 188 328 328 328 F F F 15 300-319 188 188 188 331 331 331 F F F 16 320-339 188 188 188 331 331 331 F F F 17 340-359 189 189 189 332 332 332 F F F 18 360-379 188 188 188 331 331 331 F F F 19 380-399 187 187 187 327 327 327 F F F 20 400-419 188 188 188 331 331 331 F F F 21 420-439 184 184 184 323 323 323 F F F 22 440-459 184 184 184 323 323 323 F F F 23 460-479 187 187 187 327 327 327 F F F 24 480-499 187 187 187 327 327 327 F F F 25 500-519 186 186 186 325 325 325 F F F 26 520-539 184 184 184 323 323 323 F F F 27 540-559 188 188 188 331 331 331 F F F 28 560-579 180 180 180 314 314 314 F F F 29 580-599 187 187 187 327 327 327 F F F Max gen: 189, Max flush: 332 Begin loop 95 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:11.781288913Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1896,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:11.789653309Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1896,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:11.789975502Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1896} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:11.790003048Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1896} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:11.790011717Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1896} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:11.790023375Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1896} {"msg":"Crucible 3b72e978-07b3-4f83-a5b5-f08932253ffc has session id: e44fb410-0ea8-423f-a6a1-11ce0f4a074a","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:11.790389691Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1896,"session_id":"e44fb410-0ea8-423f-a6a1-11ce0f4a074a"} {"msg":"Upstairs opts: Upstairs UUID: 3b72e978-07b3-4f83-a5b5-f08932253ffc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:11.790433894Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1896,"session_id":"e44fb410-0ea8-423f-a6a1-11ce0f4a074a"} {"msg":"Crucible stats registered with UUID: 3b72e978-07b3-4f83-a5b5-f08932253ffc","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:11.790444703Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1896,"session_id":"e44fb410-0ea8-423f-a6a1-11ce0f4a074a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T19:00:11.790453841Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1896,"session_id":"e44fb410-0ea8-423f-a6a1-11ce0f4a074a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:11.790605619Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1896,"":"io task","client":"2","":"downstairs","session_id":"e44fb410-0ea8-423f-a6a1-11ce0f4a074a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:11.790634965Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1896,"":"io task","client":"0","":"downstairs","session_id":"e44fb410-0ea8-423f-a6a1-11ce0f4a074a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:11.790645813Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1896,"":"io task","client":"1","":"downstairs","session_id":"e44fb410-0ea8-423f-a6a1-11ce0f4a074a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:11.790769305Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1896,"task":"crutest"} {"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:11.790786992Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1896} {"msg":"3b72e978-07b3-4f83-a5b5-f08932253ffc active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:11.790813908Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1896,"session_id":"e44fb410-0ea8-423f-a6a1-11ce0f4a074a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:11.790825156Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1896,"client":"0","":"downstairs","session_id":"e44fb410-0ea8-423f-a6a1-11ce0f4a074a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:11.790835055Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1896,"client":"1","":"downstairs","session_id":"e44fb410-0ea8-423f-a6a1-11ce0f4a074a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:11.790844834Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1896,"client":"2","":"downstairs","session_id":"e44fb410-0ea8-423f-a6a1-11ce0f4a074a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:11.790854462Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1896,"":"io task","client":"2","":"downstairs","session_id":"e44fb410-0ea8-423f-a6a1-11ce0f4a074a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:11.790864501Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1896,"":"io task","client":"0","":"downstairs","session_id":"e44fb410-0ea8-423f-a6a1-11ce0f4a074a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:11.790874479Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1896,"":"io task","client":"1","":"downstairs","session_id":"e44fb410-0ea8-423f-a6a1-11ce0f4a074a"} {"msg":"ds_connection connected from Ok(127.0.0.1:41736)","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:11.790958717Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1896,"":"io task","client":"1","":"downstairs","session_id":"e44fb410-0ea8-423f-a6a1-11ce0f4a074a"} {"msg":"ds_connection connected from Ok(127.0.0.1:39091)","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:11.790984583Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1896,"":"io task","client":"0","":"downstairs","session_id":"e44fb410-0ea8-423f-a6a1-11ce0f4a074a"} {"msg":"ds_connection connected from Ok(127.0.0.1:41957)","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:11.790995991Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1896,"":"io task","client":"2","":"downstairs","session_id":"e44fb410-0ea8-423f-a6a1-11ce0f4a074a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:11.791599072Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1896,"client":"0","":"downstairs","session_id":"e44fb410-0ea8-423f-a6a1-11ce0f4a074a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:11.791639676Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1896,"client":"2","":"downstairs","session_id":"e44fb410-0ea8-423f-a6a1-11ce0f4a074a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:11.791669192Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1896,"client":"1","":"downstairs","session_id":"e44fb410-0ea8-423f-a6a1-11ce0f4a074a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:11.791933993Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1896,"session_id":"e44fb410-0ea8-423f-a6a1-11ce0f4a074a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:11.791978386Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1896,"session_id":"e44fb410-0ea8-423f-a6a1-11ce0f4a074a"} {"msg":"[0]R flush_numbers[0..12]: [293, 326, 332, 332, 331, 331, 311, 331, 331, 331, 331, 320]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:11.792012091Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1896,"":"downstairs","":"downstairs","session_id":"e44fb410-0ea8-423f-a6a1-11ce0f4a074a"} {"msg":"[0]R generation[0..12]: [167, 186, 189, 189, 188, 188, 178, 188, 188, 188, 188, 184]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:11.792026479Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1896,"":"downstairs","":"downstairs","session_id":"e44fb410-0ea8-423f-a6a1-11ce0f4a074a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:11.792037448Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1896,"":"downstairs","":"downstairs","session_id":"e44fb410-0ea8-423f-a6a1-11ce0f4a074a"} {"msg":"[1]R flush_numbers[0..12]: [293, 326, 332, 332, 331, 331, 311, 331, 331, 331, 331, 320]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:11.792048226Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1896,"":"downstairs","":"downstairs","session_id":"e44fb410-0ea8-423f-a6a1-11ce0f4a074a"} {"msg":"[1]R generation[0..12]: [167, 186, 189, 189, 188, 188, 178, 188, 188, 188, 188, 184]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:11.792058964Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1896,"":"downstairs","":"downstairs","session_id":"e44fb410-0ea8-423f-a6a1-11ce0f4a074a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:11.792069903Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1896,"":"downstairs","":"downstairs","session_id":"e44fb410-0ea8-423f-a6a1-11ce0f4a074a"} {"msg":"[2]R flush_numbers[0..12]: [293, 326, 332, 332, 331, 331, 311, 331, 331, 331, 331, 320]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:11.792080621Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1896,"":"downstairs","":"downstairs","session_id":"e44fb410-0ea8-423f-a6a1-11ce0f4a074a"} {"msg":"[2]R generation[0..12]: [167, 186, 189, 189, 188, 188, 178, 188, 188, 188, 188, 184]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:11.79209135Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1896,"":"downstairs","":"downstairs","session_id":"e44fb410-0ea8-423f-a6a1-11ce0f4a074a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:11.792102528Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1896,"":"downstairs","":"downstairs","session_id":"e44fb410-0ea8-423f-a6a1-11ce0f4a074a"} {"msg":"Max found gen is 190","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:11.792112197Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1896,"":"downstairs","":"downstairs","session_id":"e44fb410-0ea8-423f-a6a1-11ce0f4a074a"} {"msg":"Generation requested: 190 >= found:190","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:11.792121985Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1896,"":"downstairs","":"downstairs","session_id":"e44fb410-0ea8-423f-a6a1-11ce0f4a074a"} {"msg":"Next flush: 333","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:11.792131424Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1896,"":"downstairs","":"downstairs","session_id":"e44fb410-0ea8-423f-a6a1-11ce0f4a074a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:11.792140832Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1896,"":"downstairs","":"downstairs","session_id":"e44fb410-0ea8-423f-a6a1-11ce0f4a074a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:11.792150931Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1896,"session_id":"e44fb410-0ea8-423f-a6a1-11ce0f4a074a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:11.79215977Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1896,"client":"0","":"downstairs","session_id":"e44fb410-0ea8-423f-a6a1-11ce0f4a074a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:11.792170718Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1896,"client":"1","":"downstairs","session_id":"e44fb410-0ea8-423f-a6a1-11ce0f4a074a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:11.792180547Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1896,"client":"2","":"downstairs","session_id":"e44fb410-0ea8-423f-a6a1-11ce0f4a074a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:11.792190415Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1896,"session_id":"e44fb410-0ea8-423f-a6a1-11ce0f4a074a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:11.792199474Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1896,"session_id":"e44fb410-0ea8-423f-a6a1-11ce0f4a074a"} {"msg":"3b72e978-07b3-4f83-a5b5-f08932253ffc is now active with session: e44fb410-0ea8-423f-a6a1-11ce0f4a074a","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:11.792209472Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1896,"session_id":"e44fb410-0ea8-423f-a6a1-11ce0f4a074a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:11.792218241Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1896,"session_id":"e44fb410-0ea8-423f-a6a1-11ce0f4a074a"} {"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:11.79222746Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1896} {"msg":"Activated sub_volume 3b72e978-07b3-4f83-a5b5-f08932253ffc","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:11.792235508Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1896} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:11.792243577Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1896,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:11.792524156Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1896,"task":"crutest"} 01/30 Write block 537 len 24576 data: 12 11 10 12 12 14 02/30 Read block 298 len 32768 03/30 Read block 419 len 28672 04/30 Flush 05/30 Write block 495 len 8192 data: 9 12 06/30 Read block 559 len 20480 07/30 Write block 104 len 12288 data: 14 14 11 08/30 Read block 413 len 24576 09/30 Write block 552 len 28672 data: 20 17 17 13 12 15 14 10/30 Write block 47 len 20480 data: 19 19 18 17 15 11/30 Write block 301 len 36864 data: 11 8 6 7 10 12 11 13 16 12/30 Write block 154 len 36864 data: 12 7 10 13 11 12 13 13 12 13/30 Flush 14/30 Flush 15/30 Write block 236 len 4096 data: 15 16/30 Write block 274 len 32768 data: 16 17 14 18 18 14 14 12 17/30 Read block 541 len 40960 18/30 Write block 189 len 12288 data: 13 13 20 19/30 Write block 180 len 20480 data: 14 13 13 13 15 20/30 Write block 470 len 16384 data: 17 15 14 12 21/30 Read block 452 len 24576 22/30 Write block 68 len 8192 data: 13 12 23/30 Write block 288 len 24576 data: 18 21 21 16 15 14 24/30 Write block 245 len 28672 data: 13 12 8 9 7 9 11 25/30 Write block 185 len 12288 data: 12 10 13 26/30 Read block 264 len 36864 27/30 Write block 571 len 28672 data: 13 12 12 12 12 11 12 28/30 Read block 173 len 12288 29/30 Read block 420 len 20480 30/30 Write block 91 len 16384 data: 16 17 17 14 ---------------------------------------------------------------- Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Read 8 Done Done Done false 1002 Acked Read 7 Done Done Done false 1003 Acked Flush 0 Done Sent Done false 1004 Acked Write 2 Done Sent Done false 1005 Acked Read 5 Done Sent Done false 1006 Acked Write 3 Done Sent Done false 1007 Acked Read 6 Done Sent Done false 1008 Acked Write 7 Done Sent Done false 1009 Acked Write 5 Done Sent Done false 1010 Acked Write 9 Done Sent Done false 1011 Acked Write 9 Done Sent Done false 1012 Acked Flush 0 Done Sent Done false 1013 Acked Flush 0 Done Sent Done false 1014 Acked Write 1 Done Sent Done false 1015 Acked Write 8 Done Sent Done false 1016 Acked Read 10 Done Sent Done false 1017 Acked Write 3 Done Sent Done false 1018 Acked Write 5 Done Sent Done false 1019 Acked Write 4 Done Sent Done false 1020 Acked Read 6 Done Sent Done false 1021 Acked Write 2 Done Sent Done false 1022 Acked Write 6 Done Sent Done false 1023 Acked Write 7 Done Sent Done false 1024 Acked Write 3 Done Sent Done false 1025 Acked Read 9 Done Sent Done false 1026 Acked Write 7 Done Sent Done false 1027 Acked Read 3 Done Sent Done false 1028 Acked Read 5 Done Sent Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 27 1 29 Done 29 3 29 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1013)) None Some(JobId(1013)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T19:00:11.800624451Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1896} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:11.800836359Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1896,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 293 293 293 F F F 1 020-039 186 186 186 326 326 326 F F F 2 040-059 190 189 190 334 332 334 F F F <--- 3 060-079 189 189 189 332 332 332 T F T <--- 4 080-099 188 188 188 331 331 331 T F T <--- 5 100-119 190 188 190 334 331 334 F F F <--- 6 120-139 178 178 178 311 311 311 F F F 7 140-159 190 188 190 334 331 334 F F F <--- 8 160-179 190 188 190 334 331 334 F F F <--- 9 180-199 188 188 188 331 331 331 T F T <--- 10 200-219 188 188 188 331 331 331 F F F 11 220-239 184 184 184 320 320 320 T F T <--- 12 240-259 183 183 183 319 319 319 T F T <--- 13 260-279 187 187 187 327 327 327 T F T <--- 14 280-299 188 188 188 328 328 328 T F T <--- 15 300-319 190 188 190 334 331 334 F F F <--- 16 320-339 188 188 188 331 331 331 F F F 17 340-359 189 189 189 332 332 332 F F F 18 360-379 188 188 188 331 331 331 F F F 19 380-399 187 187 187 327 327 327 F F F 20 400-419 188 188 188 331 331 331 F F F 21 420-439 184 184 184 323 323 323 F F F 22 440-459 184 184 184 323 323 323 F F F 23 460-479 187 187 187 327 327 327 T F T <--- 24 480-499 190 187 190 334 327 334 F F F <--- 25 500-519 186 186 186 325 325 325 F F F 26 520-539 190 184 190 333 323 333 F T F <--- 27 540-559 190 188 190 334 331 334 F T F <--- 28 560-579 180 180 180 314 314 314 T F T <--- 29 580-599 187 187 187 327 327 327 F F F Max gen: 190, Max flush: 334 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 95 Verifying data now No change after verify Loop: 95 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 293 293 293 F F F 1 020-039 186 186 186 326 326 326 F F F 2 040-059 190 190 190 334 334 334 F F F 3 060-079 191 191 191 335 335 335 F F F 4 080-099 191 191 191 335 335 335 F F F 5 100-119 190 190 190 334 334 334 F F F 6 120-139 178 178 178 311 311 311 F F F 7 140-159 190 190 190 334 334 334 F F F 8 160-179 190 190 190 334 334 334 F F F 9 180-199 191 191 191 335 335 335 F F F 10 200-219 188 188 188 331 331 331 F F F 11 220-239 191 191 191 335 335 335 F F F 12 240-259 191 191 191 335 335 335 F F F 13 260-279 191 191 191 335 335 335 F F F 14 280-299 191 191 191 335 335 335 F F F 15 300-319 190 190 190 334 334 334 F F F 16 320-339 188 188 188 331 331 331 F F F 17 340-359 189 189 189 332 332 332 F F F 18 360-379 188 188 188 331 331 331 F F F 19 380-399 187 187 187 327 327 327 F F F 20 400-419 188 188 188 331 331 331 F F F 21 420-439 184 184 184 323 323 323 F F F 22 440-459 184 184 184 323 323 323 F F F 23 460-479 191 191 191 335 335 335 F F F 24 480-499 190 190 190 334 334 334 F F F 25 500-519 186 186 186 325 325 325 F F F 26 520-539 190 190 190 333 333 333 F F F 27 540-559 190 190 190 334 334 334 F F F 28 560-579 191 191 191 335 335 335 F F F 29 580-599 187 187 187 327 327 327 F F F Max gen: 191, Max flush: 335 Begin loop 96 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:16.065732565Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1904,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T19:00:16.072794973Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1904,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:16.072851114Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1904,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:16.079667459Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1904,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:16.07992815Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1904} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:16.079951457Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1904} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:16.079962865Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1904} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:16.079974673Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1904} {"msg":"Crucible 6739f2d5-6754-4a0e-b6a8-8d47ea453584 has session id: 7e14e96a-66fa-4677-84bd-8395d5d2ec00","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:16.080316843Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1904,"session_id":"7e14e96a-66fa-4677-84bd-8395d5d2ec00"} {"msg":"Upstairs opts: Upstairs UUID: 6739f2d5-6754-4a0e-b6a8-8d47ea453584, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:16.080346599Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1904,"session_id":"7e14e96a-66fa-4677-84bd-8395d5d2ec00"} {"msg":"Crucible stats registered with UUID: 6739f2d5-6754-4a0e-b6a8-8d47ea453584","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:16.080357767Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1904,"session_id":"7e14e96a-66fa-4677-84bd-8395d5d2ec00"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T19:00:16.080367026Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1904,"session_id":"7e14e96a-66fa-4677-84bd-8395d5d2ec00"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:16.080486858Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1904,"":"io task","client":"1","":"downstairs","session_id":"7e14e96a-66fa-4677-84bd-8395d5d2ec00"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:16.080503655Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1904,"":"io task","client":"0","":"downstairs","session_id":"7e14e96a-66fa-4677-84bd-8395d5d2ec00"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:16.080514074Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1904,"":"io task","client":"2","":"downstairs","session_id":"7e14e96a-66fa-4677-84bd-8395d5d2ec00"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:16.0806756Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1904,"task":"crutest"} {"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:16.080694207Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1904} {"msg":"6739f2d5-6754-4a0e-b6a8-8d47ea453584 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:16.080707085Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1904,"session_id":"7e14e96a-66fa-4677-84bd-8395d5d2ec00"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:16.080716084Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1904,"client":"0","":"downstairs","session_id":"7e14e96a-66fa-4677-84bd-8395d5d2ec00"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:16.080725923Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1904,"client":"1","":"downstairs","session_id":"7e14e96a-66fa-4677-84bd-8395d5d2ec00"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:16.080735491Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1904,"client":"2","":"downstairs","session_id":"7e14e96a-66fa-4677-84bd-8395d5d2ec00"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:16.080747289Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1904,"":"io task","client":"2","":"downstairs","session_id":"7e14e96a-66fa-4677-84bd-8395d5d2ec00"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:16.080778795Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1904,"":"io task","client":"0","":"downstairs","session_id":"7e14e96a-66fa-4677-84bd-8395d5d2ec00"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:16.080856363Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1904,"":"io task","client":"1","":"downstairs","session_id":"7e14e96a-66fa-4677-84bd-8395d5d2ec00"} {"msg":"ds_connection connected from Ok(127.0.0.1:51498)","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:16.080920844Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1904,"":"io task","client":"0","":"downstairs","session_id":"7e14e96a-66fa-4677-84bd-8395d5d2ec00"} {"msg":"ds_connection connected from Ok(127.0.0.1:40250)","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:16.080942161Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1904,"":"io task","client":"1","":"downstairs","session_id":"7e14e96a-66fa-4677-84bd-8395d5d2ec00"} {"msg":"ds_connection connected from Ok(127.0.0.1:58147)","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:16.080953229Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1904,"":"io task","client":"2","":"downstairs","session_id":"7e14e96a-66fa-4677-84bd-8395d5d2ec00"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:16.081532804Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1904,"client":"1","":"downstairs","session_id":"7e14e96a-66fa-4677-84bd-8395d5d2ec00"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:16.08155589Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1904,"client":"2","":"downstairs","session_id":"7e14e96a-66fa-4677-84bd-8395d5d2ec00"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:16.081588205Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1904,"client":"0","":"downstairs","session_id":"7e14e96a-66fa-4677-84bd-8395d5d2ec00"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:16.081836179Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1904,"session_id":"7e14e96a-66fa-4677-84bd-8395d5d2ec00"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:16.081863735Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1904,"session_id":"7e14e96a-66fa-4677-84bd-8395d5d2ec00"} {"msg":"[0]R flush_numbers[0..12]: [293, 326, 334, 335, 335, 334, 311, 334, 334, 335, 331, 335]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:16.081916417Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1904,"":"downstairs","":"downstairs","session_id":"7e14e96a-66fa-4677-84bd-8395d5d2ec00"} {"msg":"[0]R generation[0..12]: [167, 186, 190, 191, 191, 190, 178, 190, 190, 191, 188, 191]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:16.081927915Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1904,"":"downstairs","":"downstairs","session_id":"7e14e96a-66fa-4677-84bd-8395d5d2ec00"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:16.081939164Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1904,"":"downstairs","":"downstairs","session_id":"7e14e96a-66fa-4677-84bd-8395d5d2ec00"} {"msg":"[1]R flush_numbers[0..12]: [293, 326, 334, 335, 335, 334, 311, 334, 334, 335, 331, 335]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:16.081949982Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1904,"":"downstairs","":"downstairs","session_id":"7e14e96a-66fa-4677-84bd-8395d5d2ec00"} {"msg":"[1]R generation[0..12]: [167, 186, 190, 191, 191, 190, 178, 190, 190, 191, 188, 191]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:16.08196277Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1904,"":"downstairs","":"downstairs","session_id":"7e14e96a-66fa-4677-84bd-8395d5d2ec00"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:16.081973829Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1904,"":"downstairs","":"downstairs","session_id":"7e14e96a-66fa-4677-84bd-8395d5d2ec00"} {"msg":"[2]R flush_numbers[0..12]: [293, 326, 334, 335, 335, 334, 311, 334, 334, 335, 331, 335]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:16.081984417Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1904,"":"downstairs","":"downstairs","session_id":"7e14e96a-66fa-4677-84bd-8395d5d2ec00"} {"msg":"[2]R generation[0..12]: [167, 186, 190, 191, 191, 190, 178, 190, 190, 191, 188, 191]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:16.081995115Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1904,"":"downstairs","":"downstairs","session_id":"7e14e96a-66fa-4677-84bd-8395d5d2ec00"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:16.082006154Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1904,"":"downstairs","":"downstairs","session_id":"7e14e96a-66fa-4677-84bd-8395d5d2ec00"} {"msg":"Max found gen is 192","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:16.082015922Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1904,"":"downstairs","":"downstairs","session_id":"7e14e96a-66fa-4677-84bd-8395d5d2ec00"} {"msg":"Generation requested: 192 >= found:192","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:16.082025831Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1904,"":"downstairs","":"downstairs","session_id":"7e14e96a-66fa-4677-84bd-8395d5d2ec00"} {"msg":"Next flush: 336","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:16.082035339Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1904,"":"downstairs","":"downstairs","session_id":"7e14e96a-66fa-4677-84bd-8395d5d2ec00"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:16.082044818Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1904,"":"downstairs","":"downstairs","session_id":"7e14e96a-66fa-4677-84bd-8395d5d2ec00"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:16.082054617Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1904,"session_id":"7e14e96a-66fa-4677-84bd-8395d5d2ec00"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:16.082063435Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1904,"client":"0","":"downstairs","session_id":"7e14e96a-66fa-4677-84bd-8395d5d2ec00"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:16.082073374Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1904,"client":"1","":"downstairs","session_id":"7e14e96a-66fa-4677-84bd-8395d5d2ec00"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:16.082083092Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1904,"client":"2","":"downstairs","session_id":"7e14e96a-66fa-4677-84bd-8395d5d2ec00"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:16.082093021Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1904,"session_id":"7e14e96a-66fa-4677-84bd-8395d5d2ec00"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:16.08210219Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1904,"session_id":"7e14e96a-66fa-4677-84bd-8395d5d2ec00"} {"msg":"6739f2d5-6754-4a0e-b6a8-8d47ea453584 is now active with session: 7e14e96a-66fa-4677-84bd-8395d5d2ec00","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:16.082113378Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1904,"session_id":"7e14e96a-66fa-4677-84bd-8395d5d2ec00"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:16.082122257Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1904,"session_id":"7e14e96a-66fa-4677-84bd-8395d5d2ec00"} {"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:16.082131355Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1904} {"msg":"Activated sub_volume 6739f2d5-6754-4a0e-b6a8-8d47ea453584","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:16.082139274Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1904} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:16.082147473Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1904,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:16.082458197Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1904,"task":"crutest"} 01/30 Write block 226 len 4096 data: 12 02/30 Write block 536 len 24576 data: 14 13 12 11 13 13 03/30 Read block 302 len 20480 04/30 Write block 575 len 12288 data: 13 12 13 05/30 Write block 227 len 28672 data: 13 15 16 16 13 13 12 06/30 Write block 217 len 8192 data: 8 7 07/30 Write block 422 len 8192 data: 14 14 08/30 Write block 223 len 12288 data: 15 16 13 09/30 Read block 171 len 32768 10/30 Read block 208 len 12288 11/30 Write block 384 len 16384 data: 14 13 13 10 12/30 Read block 205 len 8192 13/30 Read block 337 len 20480 14/30 Write block 88 len 20480 data: 17 17 17 17 18 15/30 Read block 535 len 4096 16/30 Read block 466 len 32768 17/30 Write block 569 len 12288 data: 12 13 14 18/30 Read block 359 len 16384 19/30 Read block 485 len 4096 20/30 Write block 169 len 28672 data: 11 9 9 13 14 14 15 21/30 Write block 18 len 40960 data: 12 14 15 14 17 11 16 18 19 17 22/30 Read block 553 len 24576 23/30 Read block 543 len 8192 24/30 Read block 250 len 36864 25/30 Write block 68 len 32768 data: 14 13 11 12 9 10 12 12 26/30 Flush 27/30 Write block 229 len 24576 data: 17 17 14 14 13 11 28/30 Write block 176 len 16384 data: 17 18 16 12 29/30 Write block 376 len 16384 data: 15 14 9 11 30/30 Write block 512 len 40960 data: 13 18 18 18 18 18 19 17 16 16 ---------------------------------------------------------------- Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Write 6 Done Done Done false 1002 Acked Read 5 Done Done Done false 1003 Acked Write 3 Done Done Done false 1004 Acked Write 7 Done Done Done false 1005 Acked Write 2 Sent Done Done false 1006 Acked Write 2 Sent Done Done false 1007 Acked Write 3 Sent Done Done false 1008 Acked Read 8 Sent Done Done false 1009 Acked Read 3 Sent Done Done false 1010 Acked Write 4 Sent Done Done false 1011 Acked Read 2 Sent Done Done false 1012 Acked Read 5 Sent Done Done false 1013 Acked Write 5 Sent Done Done false 1014 Acked Read 1 Sent Done Done false 1015 Acked Read 8 Sent Done Done false 1016 Acked Write 3 Sent Done Done false 1017 Acked Read 4 Sent Done Done false 1018 Acked Read 1 Sent Done Done false 1019 Acked Write 7 Sent Done Done false 1020 Acked Write 10 Sent Done Done false 1021 Acked Read 6 Sent Done Done false 1022 Acked Read 2 Sent Done Done false 1023 Acked Read 9 Sent Done Done false 1024 Acked Write 8 Sent Done Done false 1025 Acked Flush 0 Sent Done Done false 1026 Acked Write 6 Sent Done Done false 1027 Acked Write 4 Sent Sent Sent false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 25 3 3 31 Done 5 27 27 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T19:00:16.090054766Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1904} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:16.090246978Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1904,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 192 192 293 336 336 F F F <--- 1 020-039 186 192 192 326 336 336 F F F <--- 2 040-059 190 190 190 334 334 334 F F F 3 060-079 191 192 192 335 336 336 F F F <--- 4 080-099 191 192 192 335 336 336 F F F <--- 5 100-119 190 190 190 334 334 334 F F F 6 120-139 178 178 178 311 311 311 F F F 7 140-159 190 190 190 334 334 334 F F F 8 160-179 190 192 192 334 336 336 F T T <--- 9 180-199 191 191 191 335 335 335 F F F 10 200-219 188 192 192 331 336 336 F F F <--- 11 220-239 191 192 192 335 336 336 T T T <--- 12 240-259 191 191 191 335 335 335 F F F 13 260-279 191 191 191 335 335 335 F F F 14 280-299 191 191 191 335 335 335 F F F 15 300-319 190 190 190 334 334 334 F F F 16 320-339 188 188 188 331 331 331 F F F 17 340-359 189 189 189 332 332 332 F F F 18 360-379 188 188 188 331 331 331 F T T <--- 19 380-399 187 192 192 327 336 336 F F F <--- 20 400-419 188 188 188 331 331 331 F F F 21 420-439 184 192 192 323 336 336 F F F <--- 22 440-459 184 184 184 323 323 323 F F F 23 460-479 191 191 191 335 335 335 F F F 24 480-499 190 190 190 334 334 334 F F F 25 500-519 186 186 186 325 325 325 F T T <--- 26 520-539 190 192 192 333 336 336 T T T <--- 27 540-559 190 192 192 334 336 336 T F F <--- 28 560-579 191 192 192 335 336 336 T F F <--- 29 580-599 187 187 187 327 327 327 F F F Max gen: 192, Max flush: 336 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 96 Verifying data now No change after verify Loop: 96 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 192 192 192 336 336 336 F F F 1 020-039 192 192 192 336 336 336 F F F 2 040-059 190 190 190 334 334 334 F F F 3 060-079 192 192 192 336 336 336 F F F 4 080-099 192 192 192 336 336 336 F F F 5 100-119 190 190 190 334 334 334 F F F 6 120-139 178 178 178 311 311 311 F F F 7 140-159 190 190 190 334 334 334 F F F 8 160-179 193 193 193 337 337 337 F F F 9 180-199 191 191 191 335 335 335 F F F 10 200-219 192 192 192 336 336 336 F F F 11 220-239 193 193 193 337 337 337 F F F 12 240-259 191 191 191 335 335 335 F F F 13 260-279 191 191 191 335 335 335 F F F 14 280-299 191 191 191 335 335 335 F F F 15 300-319 190 190 190 334 334 334 F F F 16 320-339 188 188 188 331 331 331 F F F 17 340-359 189 189 189 332 332 332 F F F 18 360-379 193 193 193 337 337 337 F F F 19 380-399 192 192 192 336 336 336 F F F 20 400-419 188 188 188 331 331 331 F F F 21 420-439 192 192 192 336 336 336 F F F 22 440-459 184 184 184 323 323 323 F F F 23 460-479 191 191 191 335 335 335 F F F 24 480-499 190 190 190 334 334 334 F F F 25 500-519 193 193 193 337 337 337 F F F 26 520-539 193 193 193 337 337 337 F F F 27 540-559 192 192 192 336 336 336 F F F 28 560-579 192 192 192 336 336 336 F F F 29 580-599 187 187 187 327 327 327 F F F Max gen: 193, Max flush: 337 Begin loop 97 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:20.364568913Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1912,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:20.372841083Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1912,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:20.373135079Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1912} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:20.373159076Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1912} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:20.373167785Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1912} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:20.373179313Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1912} {"msg":"Crucible ef2e8986-396d-40c5-a651-29f87af70ee2 has session id: 363cb1bd-e34e-43fc-9b6a-baf8db31b79d","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:20.373549518Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1912,"session_id":"363cb1bd-e34e-43fc-9b6a-baf8db31b79d"} {"msg":"Upstairs opts: Upstairs UUID: ef2e8986-396d-40c5-a651-29f87af70ee2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:20.373573415Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1912,"session_id":"363cb1bd-e34e-43fc-9b6a-baf8db31b79d"} {"msg":"Crucible stats registered with UUID: ef2e8986-396d-40c5-a651-29f87af70ee2","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:20.373585473Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1912,"session_id":"363cb1bd-e34e-43fc-9b6a-baf8db31b79d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T19:00:20.373594542Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1912,"session_id":"363cb1bd-e34e-43fc-9b6a-baf8db31b79d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:20.373748509Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1912,"":"io task","client":"2","":"downstairs","session_id":"363cb1bd-e34e-43fc-9b6a-baf8db31b79d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:20.373778284Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1912,"":"io task","client":"0","":"downstairs","session_id":"363cb1bd-e34e-43fc-9b6a-baf8db31b79d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:20.373789373Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1912,"":"io task","client":"1","":"downstairs","session_id":"363cb1bd-e34e-43fc-9b6a-baf8db31b79d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:20.373899717Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1912,"task":"crutest"} {"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:20.373910575Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1912} {"msg":"ef2e8986-396d-40c5-a651-29f87af70ee2 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:20.37394709Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1912,"session_id":"363cb1bd-e34e-43fc-9b6a-baf8db31b79d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:20.373956868Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1912,"client":"0","":"downstairs","session_id":"363cb1bd-e34e-43fc-9b6a-baf8db31b79d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:20.373966717Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1912,"client":"1","":"downstairs","session_id":"363cb1bd-e34e-43fc-9b6a-baf8db31b79d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:20.373976485Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1912,"client":"2","":"downstairs","session_id":"363cb1bd-e34e-43fc-9b6a-baf8db31b79d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:20.373986144Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1912,"":"io task","client":"2","":"downstairs","session_id":"363cb1bd-e34e-43fc-9b6a-baf8db31b79d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:20.373996302Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1912,"":"io task","client":"0","":"downstairs","session_id":"363cb1bd-e34e-43fc-9b6a-baf8db31b79d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:20.374006391Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1912,"":"io task","client":"1","":"downstairs","session_id":"363cb1bd-e34e-43fc-9b6a-baf8db31b79d"} {"msg":"ds_connection connected from Ok(127.0.0.1:41735)","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:20.374065432Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1912,"":"io task","client":"2","":"downstairs","session_id":"363cb1bd-e34e-43fc-9b6a-baf8db31b79d"} {"msg":"ds_connection connected from Ok(127.0.0.1:64514)","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:20.374142311Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1912,"":"io task","client":"1","":"downstairs","session_id":"363cb1bd-e34e-43fc-9b6a-baf8db31b79d"} {"msg":"ds_connection connected from Ok(127.0.0.1:65054)","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:20.374190064Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1912,"":"io task","client":"0","":"downstairs","session_id":"363cb1bd-e34e-43fc-9b6a-baf8db31b79d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:20.374801034Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1912,"client":"0","":"downstairs","session_id":"363cb1bd-e34e-43fc-9b6a-baf8db31b79d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:20.37482451Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1912,"client":"1","":"downstairs","session_id":"363cb1bd-e34e-43fc-9b6a-baf8db31b79d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:20.37495945Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1912,"client":"2","":"downstairs","session_id":"363cb1bd-e34e-43fc-9b6a-baf8db31b79d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:20.375151852Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1912,"session_id":"363cb1bd-e34e-43fc-9b6a-baf8db31b79d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:20.3751615Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1912,"session_id":"363cb1bd-e34e-43fc-9b6a-baf8db31b79d"} {"msg":"[0]R flush_numbers[0..12]: [336, 336, 334, 336, 336, 334, 311, 334, 337, 335, 336, 337]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:20.375282323Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1912,"":"downstairs","":"downstairs","session_id":"363cb1bd-e34e-43fc-9b6a-baf8db31b79d"} {"msg":"[0]R generation[0..12]: [192, 192, 190, 192, 192, 190, 178, 190, 193, 191, 192, 193]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:20.375307909Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1912,"":"downstairs","":"downstairs","session_id":"363cb1bd-e34e-43fc-9b6a-baf8db31b79d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:20.375319947Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1912,"":"downstairs","":"downstairs","session_id":"363cb1bd-e34e-43fc-9b6a-baf8db31b79d"} {"msg":"[1]R flush_numbers[0..12]: [336, 336, 334, 336, 336, 334, 311, 334, 337, 335, 336, 337]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:20.375330735Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1912,"":"downstairs","":"downstairs","session_id":"363cb1bd-e34e-43fc-9b6a-baf8db31b79d"} {"msg":"[1]R generation[0..12]: [192, 192, 190, 192, 192, 190, 178, 190, 193, 191, 192, 193]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:20.375341354Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1912,"":"downstairs","":"downstairs","session_id":"363cb1bd-e34e-43fc-9b6a-baf8db31b79d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:20.375352422Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1912,"":"downstairs","":"downstairs","session_id":"363cb1bd-e34e-43fc-9b6a-baf8db31b79d"} {"msg":"[2]R flush_numbers[0..12]: [336, 336, 334, 336, 336, 334, 311, 334, 337, 335, 336, 337]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:20.375363071Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1912,"":"downstairs","":"downstairs","session_id":"363cb1bd-e34e-43fc-9b6a-baf8db31b79d"} {"msg":"[2]R generation[0..12]: [192, 192, 190, 192, 192, 190, 178, 190, 193, 191, 192, 193]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:20.375373869Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1912,"":"downstairs","":"downstairs","session_id":"363cb1bd-e34e-43fc-9b6a-baf8db31b79d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:20.375386937Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1912,"":"downstairs","":"downstairs","session_id":"363cb1bd-e34e-43fc-9b6a-baf8db31b79d"} {"msg":"Max found gen is 194","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:20.375396696Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1912,"":"downstairs","":"downstairs","session_id":"363cb1bd-e34e-43fc-9b6a-baf8db31b79d"} {"msg":"Generation requested: 194 >= found:194","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:20.375406514Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1912,"":"downstairs","":"downstairs","session_id":"363cb1bd-e34e-43fc-9b6a-baf8db31b79d"} {"msg":"Next flush: 338","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:20.375416003Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1912,"":"downstairs","":"downstairs","session_id":"363cb1bd-e34e-43fc-9b6a-baf8db31b79d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:20.375425461Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1912,"":"downstairs","":"downstairs","session_id":"363cb1bd-e34e-43fc-9b6a-baf8db31b79d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:20.37543536Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1912,"session_id":"363cb1bd-e34e-43fc-9b6a-baf8db31b79d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:20.375444509Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1912,"client":"0","":"downstairs","session_id":"363cb1bd-e34e-43fc-9b6a-baf8db31b79d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:20.375454327Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1912,"client":"1","":"downstairs","session_id":"363cb1bd-e34e-43fc-9b6a-baf8db31b79d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:20.375463986Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1912,"client":"2","":"downstairs","session_id":"363cb1bd-e34e-43fc-9b6a-baf8db31b79d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:20.375473884Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1912,"session_id":"363cb1bd-e34e-43fc-9b6a-baf8db31b79d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:20.375482893Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1912,"session_id":"363cb1bd-e34e-43fc-9b6a-baf8db31b79d"} {"msg":"ef2e8986-396d-40c5-a651-29f87af70ee2 is now active with session: 363cb1bd-e34e-43fc-9b6a-baf8db31b79d","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:20.375492932Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1912,"session_id":"363cb1bd-e34e-43fc-9b6a-baf8db31b79d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:20.37550164Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1912,"session_id":"363cb1bd-e34e-43fc-9b6a-baf8db31b79d"} {"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:20.375510719Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1912} {"msg":"Activated sub_volume ef2e8986-396d-40c5-a651-29f87af70ee2","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:20.375518808Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1912} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:20.375527036Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1912,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:20.375860667Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1912,"task":"crutest"} 01/30 Write block 132 len 36864 data: 12 15 17 16 14 12 14 15 18 02/30 Flush 03/30 Write block 349 len 20480 data: 7 9 11 11 10 04/30 Write block 39 len 4096 data: 20 05/30 Read block 463 len 32768 06/30 Write block 586 len 40960 data: 17 19 16 15 16 17 18 19 17 15 07/30 Read block 60 len 24576 08/30 Read block 324 len 8192 09/30 Write block 497 len 40960 data: 13 14 17 16 15 17 20 16 15 11 10/30 Read block 355 len 8192 11/30 Write block 35 len 24576 data: 17 17 15 17 21 19 12/30 Read block 231 len 24576 13/30 Write block 513 len 20480 data: 19 19 19 19 19 14/30 Read block 56 len 8192 15/30 Write block 484 len 8192 data: 11 15 16/30 Write block 164 len 40960 data: 14 16 13 12 10 12 10 10 14 15 17/30 Flush 18/30 Write block 396 len 12288 data: 13 14 13 19/30 Write block 247 len 20480 data: 9 10 8 10 12 20/30 Write block 219 len 32768 data: 7 10 15 16 16 17 14 13 21/30 Read block 204 len 28672 22/30 Write block 384 len 28672 data: 15 14 14 11 9 6 8 23/30 Read block 522 len 12288 24/30 Read block 99 len 20480 25/30 Read block 145 len 32768 26/30 Read block 272 len 12288 27/30 Write block 501 len 20480 data: 16 18 21 17 16 28/30 Write block 270 len 4096 data: 16 29/30 Write block 209 len 28672 data: 20 17 13 15 14 14 13 30/30 Read block 105 len 32768 ---------------------------------------------------------------- Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 5 Done Done Sent false 1003 Acked Write 1 Done Done Sent false 1004 Acked Read 8 Done Done Sent false 1005 Acked Write 10 Done Done Sent false 1006 Acked Read 6 Done Done Sent false 1007 Acked Read 2 Done Done Sent false 1008 Acked Write 10 Done Done Sent false 1009 Acked Read 2 Done Done Sent false 1010 Acked Write 6 Done Done Sent false 1011 Acked Read 6 Done Done Sent false 1012 Acked Write 5 Done Done Sent false 1013 Acked Read 2 Done Done Sent false 1014 Acked Write 2 Done Done Sent false 1015 Acked Write 10 Done Done Sent false 1016 Acked Flush 0 Done Done Sent false 1017 Acked Write 3 Done Done Sent false 1018 Acked Write 5 Done Done Sent false 1019 Acked Write 8 Done Done Sent false 1020 Acked Read 7 Done Done Sent false 1021 Acked Write 7 Done Done Sent false 1022 Acked Read 3 Done Done Sent false 1023 Acked Read 5 Done Done Sent false 1024 Acked Read 8 Done Done Sent false 1025 Acked Read 3 Done Done Sent false 1026 Acked Write 5 Done Done Sent false 1027 Acked Write 1 Done Done Sent false 1028 Acked Write 7 Done Done Sent false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 28 29 Done 28 27 0 55 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) Some(JobId(1001)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T19:00:20.385051771Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1912} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:20.385256661Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1912,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 192 192 192 336 336 336 F F F 1 020-039 194 194 192 339 339 336 F F F <--- 2 040-059 194 194 190 339 339 334 F F F <--- 3 060-079 192 192 192 336 336 336 F F F 4 080-099 192 192 192 336 336 336 F F F 5 100-119 190 190 190 334 334 334 F F F 6 120-139 194 194 194 338 338 338 F F F 7 140-159 194 194 194 338 338 338 F F F 8 160-179 194 194 193 339 339 337 F F F <--- 9 180-199 191 191 191 335 335 335 F F F 10 200-219 192 192 192 336 336 336 T T F <--- 11 220-239 193 193 193 337 337 337 T T F <--- 12 240-259 191 191 191 335 335 335 T T F <--- 13 260-279 191 191 191 335 335 335 T T F <--- 14 280-299 191 191 191 335 335 335 F F F 15 300-319 190 190 190 334 334 334 F F F 16 320-339 188 188 188 331 331 331 F F F 17 340-359 194 194 189 339 339 332 F F F <--- 18 360-379 193 193 193 337 337 337 F F F 19 380-399 192 192 192 336 336 336 T T F <--- 20 400-419 188 188 188 331 331 331 F F F 21 420-439 192 192 192 336 336 336 F F F 22 440-459 184 184 184 323 323 323 F F F 23 460-479 191 191 191 335 335 335 F F F 24 480-499 194 194 190 339 339 334 F F F <--- 25 500-519 194 194 193 339 339 337 T T F <--- 26 520-539 193 193 193 337 337 337 F F F 27 540-559 192 192 192 336 336 336 F F F 28 560-579 192 192 192 336 336 336 F F F 29 580-599 194 194 187 339 339 327 F F F <--- Max gen: 194, Max flush: 339 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 97 Verifying data now No change after verify Loop: 97 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 192 192 192 336 336 336 F F F 1 020-039 194 194 194 339 339 339 F F F 2 040-059 194 194 194 339 339 339 F F F 3 060-079 192 192 192 336 336 336 F F F 4 080-099 192 192 192 336 336 336 F F F 5 100-119 190 190 190 334 334 334 F F F 6 120-139 194 194 194 338 338 338 F F F 7 140-159 194 194 194 338 338 338 F F F 8 160-179 194 194 194 339 339 339 F F F 9 180-199 191 191 191 335 335 335 F F F 10 200-219 195 195 195 340 340 340 F F F 11 220-239 195 195 195 340 340 340 F F F 12 240-259 195 195 195 340 340 340 F F F 13 260-279 195 195 195 340 340 340 F F F 14 280-299 191 191 191 335 335 335 F F F 15 300-319 190 190 190 334 334 334 F F F 16 320-339 188 188 188 331 331 331 F F F 17 340-359 194 194 194 339 339 339 F F F 18 360-379 193 193 193 337 337 337 F F F 19 380-399 195 195 195 340 340 340 F F F 20 400-419 188 188 188 331 331 331 F F F 21 420-439 192 192 192 336 336 336 F F F 22 440-459 184 184 184 323 323 323 F F F 23 460-479 191 191 191 335 335 335 F F F 24 480-499 194 194 194 339 339 339 F F F 25 500-519 195 195 195 340 340 340 F F F 26 520-539 193 193 193 337 337 337 F F F 27 540-559 192 192 192 336 336 336 F F F 28 560-579 192 192 192 336 336 336 F F F 29 580-599 194 194 194 339 339 339 F F F Max gen: 195, Max flush: 340 Begin loop 98 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:24.635233628Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1920,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:24.643327454Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1920,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:24.643620541Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1920} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:24.643648437Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1920} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:24.643657245Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1920} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:24.643668774Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1920} {"msg":"Crucible c917f97a-558a-434f-b358-8b38f3d7bc4e has session id: 743e7189-6cfb-4c8c-a8e5-1d081013ab0f","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:24.64403109Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1920,"session_id":"743e7189-6cfb-4c8c-a8e5-1d081013ab0f"} {"msg":"Upstairs opts: Upstairs UUID: c917f97a-558a-434f-b358-8b38f3d7bc4e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:24.644062236Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1920,"session_id":"743e7189-6cfb-4c8c-a8e5-1d081013ab0f"} {"msg":"Crucible stats registered with UUID: c917f97a-558a-434f-b358-8b38f3d7bc4e","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:24.644073664Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1920,"session_id":"743e7189-6cfb-4c8c-a8e5-1d081013ab0f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T19:00:24.644083162Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1920,"session_id":"743e7189-6cfb-4c8c-a8e5-1d081013ab0f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:24.644211344Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1920,"":"io task","client":"2","":"downstairs","session_id":"743e7189-6cfb-4c8c-a8e5-1d081013ab0f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:24.644228201Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1920,"":"io task","client":"0","":"downstairs","session_id":"743e7189-6cfb-4c8c-a8e5-1d081013ab0f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:24.64423844Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1920,"":"io task","client":"1","":"downstairs","session_id":"743e7189-6cfb-4c8c-a8e5-1d081013ab0f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:24.644408894Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1920,"task":"crutest"} {"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:24.644425102Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1920} {"msg":"c917f97a-558a-434f-b358-8b38f3d7bc4e active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:24.644449238Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1920,"session_id":"743e7189-6cfb-4c8c-a8e5-1d081013ab0f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:24.644458647Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1920,"client":"0","":"downstairs","session_id":"743e7189-6cfb-4c8c-a8e5-1d081013ab0f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:24.644468386Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1920,"client":"1","":"downstairs","session_id":"743e7189-6cfb-4c8c-a8e5-1d081013ab0f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:24.644478064Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1920,"client":"2","":"downstairs","session_id":"743e7189-6cfb-4c8c-a8e5-1d081013ab0f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:24.644487863Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1920,"":"io task","client":"2","":"downstairs","session_id":"743e7189-6cfb-4c8c-a8e5-1d081013ab0f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:24.644498101Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1920,"":"io task","client":"0","":"downstairs","session_id":"743e7189-6cfb-4c8c-a8e5-1d081013ab0f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:24.64450786Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1920,"":"io task","client":"1","":"downstairs","session_id":"743e7189-6cfb-4c8c-a8e5-1d081013ab0f"} {"msg":"ds_connection connected from Ok(127.0.0.1:63863)","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:24.644563592Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1920,"":"io task","client":"1","":"downstairs","session_id":"743e7189-6cfb-4c8c-a8e5-1d081013ab0f"} {"msg":"ds_connection connected from Ok(127.0.0.1:50373)","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:24.64457497Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1920,"":"io task","client":"0","":"downstairs","session_id":"743e7189-6cfb-4c8c-a8e5-1d081013ab0f"} {"msg":"ds_connection connected from Ok(127.0.0.1:64807)","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:24.644587998Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1920,"":"io task","client":"2","":"downstairs","session_id":"743e7189-6cfb-4c8c-a8e5-1d081013ab0f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:24.64525356Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1920,"client":"0","":"downstairs","session_id":"743e7189-6cfb-4c8c-a8e5-1d081013ab0f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:24.645268578Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1920,"client":"2","":"downstairs","session_id":"743e7189-6cfb-4c8c-a8e5-1d081013ab0f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:24.645282485Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1920,"client":"1","":"downstairs","session_id":"743e7189-6cfb-4c8c-a8e5-1d081013ab0f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:24.645640773Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1920,"session_id":"743e7189-6cfb-4c8c-a8e5-1d081013ab0f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:24.645650511Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1920,"session_id":"743e7189-6cfb-4c8c-a8e5-1d081013ab0f"} {"msg":"[0]R flush_numbers[0..12]: [336, 339, 339, 336, 336, 334, 338, 338, 339, 335, 340, 340]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:24.645675697Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1920,"":"downstairs","":"downstairs","session_id":"743e7189-6cfb-4c8c-a8e5-1d081013ab0f"} {"msg":"[0]R generation[0..12]: [192, 194, 194, 192, 192, 190, 194, 194, 194, 191, 195, 195]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:24.645686916Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1920,"":"downstairs","":"downstairs","session_id":"743e7189-6cfb-4c8c-a8e5-1d081013ab0f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:24.645697724Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1920,"":"downstairs","":"downstairs","session_id":"743e7189-6cfb-4c8c-a8e5-1d081013ab0f"} {"msg":"[1]R flush_numbers[0..12]: [336, 339, 339, 336, 336, 334, 338, 338, 339, 335, 340, 340]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:24.645708283Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1920,"":"downstairs","":"downstairs","session_id":"743e7189-6cfb-4c8c-a8e5-1d081013ab0f"} {"msg":"[1]R generation[0..12]: [192, 194, 194, 192, 192, 190, 194, 194, 194, 191, 195, 195]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:24.645718891Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1920,"":"downstairs","":"downstairs","session_id":"743e7189-6cfb-4c8c-a8e5-1d081013ab0f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:24.645729889Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1920,"":"downstairs","":"downstairs","session_id":"743e7189-6cfb-4c8c-a8e5-1d081013ab0f"} {"msg":"[2]R flush_numbers[0..12]: [336, 339, 339, 336, 336, 334, 338, 338, 339, 335, 340, 340]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:24.645740568Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1920,"":"downstairs","":"downstairs","session_id":"743e7189-6cfb-4c8c-a8e5-1d081013ab0f"} {"msg":"[2]R generation[0..12]: [192, 194, 194, 192, 192, 190, 194, 194, 194, 191, 195, 195]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:24.645750896Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1920,"":"downstairs","":"downstairs","session_id":"743e7189-6cfb-4c8c-a8e5-1d081013ab0f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:24.645761915Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1920,"":"downstairs","":"downstairs","session_id":"743e7189-6cfb-4c8c-a8e5-1d081013ab0f"} {"msg":"Max found gen is 196","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:24.645771503Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1920,"":"downstairs","":"downstairs","session_id":"743e7189-6cfb-4c8c-a8e5-1d081013ab0f"} {"msg":"Generation requested: 196 >= found:196","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:24.645781272Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1920,"":"downstairs","":"downstairs","session_id":"743e7189-6cfb-4c8c-a8e5-1d081013ab0f"} {"msg":"Next flush: 341","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:24.645790581Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1920,"":"downstairs","":"downstairs","session_id":"743e7189-6cfb-4c8c-a8e5-1d081013ab0f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:24.645800049Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1920,"":"downstairs","":"downstairs","session_id":"743e7189-6cfb-4c8c-a8e5-1d081013ab0f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:24.645811447Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1920,"session_id":"743e7189-6cfb-4c8c-a8e5-1d081013ab0f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:24.645820256Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1920,"client":"0","":"downstairs","session_id":"743e7189-6cfb-4c8c-a8e5-1d081013ab0f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:24.645830015Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1920,"client":"1","":"downstairs","session_id":"743e7189-6cfb-4c8c-a8e5-1d081013ab0f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:24.645839673Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1920,"client":"2","":"downstairs","session_id":"743e7189-6cfb-4c8c-a8e5-1d081013ab0f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:24.645849542Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1920,"session_id":"743e7189-6cfb-4c8c-a8e5-1d081013ab0f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:24.64585871Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1920,"session_id":"743e7189-6cfb-4c8c-a8e5-1d081013ab0f"} {"msg":"c917f97a-558a-434f-b358-8b38f3d7bc4e is now active with session: 743e7189-6cfb-4c8c-a8e5-1d081013ab0f","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:24.645868729Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1920,"session_id":"743e7189-6cfb-4c8c-a8e5-1d081013ab0f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:24.645877478Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1920,"session_id":"743e7189-6cfb-4c8c-a8e5-1d081013ab0f"} {"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:24.645886636Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1920} {"msg":"Activated sub_volume c917f97a-558a-434f-b358-8b38f3d7bc4e","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:24.645894605Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1920} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:24.645902844Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1920,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:24.646233445Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1920,"task":"crutest"} 01/30 Write block 62 len 36864 data: 13 14 14 12 12 15 15 14 12 02/30 Write block 538 len 40960 data: 13 12 14 14 15 12 13 14 16 16 03/30 Write block 456 len 40960 data: 16 19 13 14 13 13 12 12 14 14 04/30 Read block 595 len 16384 05/30 Read block 494 len 16384 06/30 Read block 73 len 28672 07/30 Write block 537 len 40960 data: 14 14 13 15 15 16 13 14 15 17 08/30 Read block 125 len 28672 09/30 Read block 332 len 40960 10/30 Write block 203 len 36864 data: 14 15 16 18 24 22 21 18 14 11/30 Read block 218 len 20480 12/30 Read block 304 len 12288 13/30 Read block 408 len 36864 14/30 Write block 201 len 8192 data: 10 11 15/30 Read block 126 len 40960 16/30 Flush 17/30 Write block 25 len 8192 data: 19 20 18/30 Write block 506 len 24576 data: 12 11 9 8 14 13 19/30 Read block 246 len 16384 20/30 Write block 52 len 16384 data: 13 13 17 15 21/30 Flush 22/30 Write block 251 len 12288 data: 13 15 17 23/30 Write block 252 len 32768 data: 16 18 20 18 20 22 24 21 24/30 Read block 141 len 4096 25/30 Read block 335 len 12288 26/30 Flush 27/30 Write block 497 len 28672 data: 14 15 18 17 17 19 22 28/30 Write block 302 len 16384 data: 9 7 8 11 29/30 Write block 295 len 32768 data: 7 4 4 4 5 8 12 10 30/30 Write block 458 len 16384 data: 14 15 14 14 ---------------------------------------------------------------- Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Write 10 Done Sent Done false 1002 Acked Write 10 Done Sent Done false 1003 Acked Read 4 Done Sent Done false 1004 Acked Read 4 Done Sent Done false 1005 Acked Read 7 Done Sent Done false 1006 Acked Write 10 Done Sent Done false 1007 Acked Read 7 Done Sent Done false 1008 Acked Read 10 Done Sent Done false 1009 Acked Write 9 Done Sent Done false 1010 Acked Read 5 Done Sent Done false 1011 Acked Read 3 Done Sent Done false 1012 Acked Read 9 Done Sent Done false 1013 Acked Write 2 Done Sent Done false 1014 Acked Read 10 Done Sent Done false 1015 Acked Flush 0 Done Sent Done false 1016 Acked Write 2 Done Sent Done false 1017 Acked Write 6 Done Sent Done false 1018 Acked Read 4 Done Sent Done false 1019 Acked Write 4 Done Sent Done false 1020 Acked Flush 0 Done Sent Done false 1021 Acked Write 3 Done Sent Done false 1022 Acked Write 8 Done Sent Done false 1023 Acked Read 1 Done Sent Done false 1024 Acked Read 3 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 7 Sent Sent Sent false 1027 Acked Write 4 Sent Sent Sent false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 4 29 4 37 Done 26 1 26 53 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) None Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T19:00:24.654685908Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1920} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:24.654870711Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1920,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 192 192 192 336 336 336 F F F 1 020-039 196 194 196 342 339 342 F F F <--- 2 040-059 196 194 196 342 339 342 F F F <--- 3 060-079 196 192 196 341 336 341 F T F <--- 4 080-099 192 192 192 336 336 336 F F F 5 100-119 190 190 190 334 334 334 F F F 6 120-139 194 194 194 338 338 338 F F F 7 140-159 194 194 194 338 338 338 F F F 8 160-179 194 194 194 339 339 339 F F F 9 180-199 191 191 191 335 335 335 F F F 10 200-219 196 195 196 341 340 341 F F F <--- 11 220-239 195 195 195 340 340 340 F F F 12 240-259 196 195 196 343 340 343 F F F <--- 13 260-279 195 195 195 340 340 340 F F F 14 280-299 191 191 191 335 335 335 T F T <--- 15 300-319 190 190 190 334 334 334 T F T <--- 16 320-339 188 188 188 331 331 331 F F F 17 340-359 194 194 194 339 339 339 F F F 18 360-379 193 193 193 337 337 337 F F F 19 380-399 195 195 195 340 340 340 F F F 20 400-419 188 188 188 331 331 331 F F F 21 420-439 192 192 192 336 336 336 F F F 22 440-459 196 184 196 341 323 341 T F T <--- 23 460-479 196 191 196 341 335 341 T F T <--- 24 480-499 194 194 194 339 339 339 T F T <--- 25 500-519 196 195 196 342 340 342 T F T <--- 26 520-539 196 193 196 341 337 341 F F F <--- 27 540-559 196 192 196 341 336 341 F F F <--- 28 560-579 192 192 192 336 336 336 F F F 29 580-599 194 194 194 339 339 339 F F F Max gen: 196, Max flush: 343 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 98 Verifying data now No change after verify Loop: 98 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 192 192 192 336 336 336 F F F 1 020-039 196 196 196 342 342 342 F F F 2 040-059 196 196 196 342 342 342 F F F 3 060-079 196 196 196 341 341 341 F F F 4 080-099 192 192 192 336 336 336 F F F 5 100-119 190 190 190 334 334 334 F F F 6 120-139 194 194 194 338 338 338 F F F 7 140-159 194 194 194 338 338 338 F F F 8 160-179 194 194 194 339 339 339 F F F 9 180-199 191 191 191 335 335 335 F F F 10 200-219 196 196 196 341 341 341 F F F 11 220-239 195 195 195 340 340 340 F F F 12 240-259 196 196 196 343 343 343 F F F 13 260-279 195 195 195 340 340 340 F F F 14 280-299 197 197 197 344 344 344 F F F 15 300-319 197 197 197 344 344 344 F F F 16 320-339 188 188 188 331 331 331 F F F 17 340-359 194 194 194 339 339 339 F F F 18 360-379 193 193 193 337 337 337 F F F 19 380-399 195 195 195 340 340 340 F F F 20 400-419 188 188 188 331 331 331 F F F 21 420-439 192 192 192 336 336 336 F F F 22 440-459 197 197 197 344 344 344 F F F 23 460-479 197 197 197 344 344 344 F F F 24 480-499 197 197 197 344 344 344 F F F 25 500-519 197 197 197 344 344 344 F F F 26 520-539 196 196 196 341 341 341 F F F 27 540-559 196 196 196 341 341 341 F F F 28 560-579 192 192 192 336 336 336 F F F 29 580-599 194 194 194 339 339 339 F F F Max gen: 197, Max flush: 344 Begin loop 99 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:28.907216629Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1928,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T19:00:28.914320631Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1928,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:28.914376542Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1928,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:28.921204835Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1928,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:28.921473125Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1928} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"771537701e93e34aa28534fa5d7d0b08164218fe\",\n git_commit_timestamp: \"2026-07-28T18:37:56.000000000Z\",\n git_branch: \"alan/improve-a-log\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:28.921498522Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1928} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:28.921507Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1928} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:28.921518919Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1928} {"msg":"Crucible 60462765-1256-48a1-b5e6-a2ca8160270b has session id: 89bca040-7d33-40cd-9378-9f60d363c4b6","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:28.921840581Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1928,"session_id":"89bca040-7d33-40cd-9378-9f60d363c4b6"} {"msg":"Upstairs opts: Upstairs UUID: 60462765-1256-48a1-b5e6-a2ca8160270b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:28.921870737Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1928,"session_id":"89bca040-7d33-40cd-9378-9f60d363c4b6"} {"msg":"Crucible stats registered with UUID: 60462765-1256-48a1-b5e6-a2ca8160270b","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:28.921881885Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1928,"session_id":"89bca040-7d33-40cd-9378-9f60d363c4b6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T19:00:28.921890924Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1928,"session_id":"89bca040-7d33-40cd-9378-9f60d363c4b6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:28.922008906Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1928,"":"io task","client":"2","":"downstairs","session_id":"89bca040-7d33-40cd-9378-9f60d363c4b6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:28.922025404Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1928,"":"io task","client":"1","":"downstairs","session_id":"89bca040-7d33-40cd-9378-9f60d363c4b6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:28.922035732Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1928,"":"io task","client":"0","":"downstairs","session_id":"89bca040-7d33-40cd-9378-9f60d363c4b6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:28.9221861Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1928,"task":"crutest"} {"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:28.922196489Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1928} {"msg":"60462765-1256-48a1-b5e6-a2ca8160270b active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:28.922213176Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1928,"session_id":"89bca040-7d33-40cd-9378-9f60d363c4b6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:28.922221905Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1928,"client":"0","":"downstairs","session_id":"89bca040-7d33-40cd-9378-9f60d363c4b6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:28.922233743Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1928,"client":"1","":"downstairs","session_id":"89bca040-7d33-40cd-9378-9f60d363c4b6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:28.922243612Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1928,"client":"2","":"downstairs","session_id":"89bca040-7d33-40cd-9378-9f60d363c4b6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:28.92225322Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1928,"":"io task","client":"2","":"downstairs","session_id":"89bca040-7d33-40cd-9378-9f60d363c4b6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:28.922263609Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1928,"":"io task","client":"0","":"downstairs","session_id":"89bca040-7d33-40cd-9378-9f60d363c4b6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:28.922273667Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1928,"":"io task","client":"1","":"downstairs","session_id":"89bca040-7d33-40cd-9378-9f60d363c4b6"} {"msg":"ds_connection connected from Ok(127.0.0.1:49894)","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:28.92232054Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1928,"":"io task","client":"2","":"downstairs","session_id":"89bca040-7d33-40cd-9378-9f60d363c4b6"} {"msg":"ds_connection connected from Ok(127.0.0.1:43392)","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:28.922331579Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1928,"":"io task","client":"1","":"downstairs","session_id":"89bca040-7d33-40cd-9378-9f60d363c4b6"} {"msg":"ds_connection connected from Ok(127.0.0.1:53026)","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:28.922422725Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1928,"":"io task","client":"0","":"downstairs","session_id":"89bca040-7d33-40cd-9378-9f60d363c4b6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:28.922945218Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1928,"client":"2","":"downstairs","session_id":"89bca040-7d33-40cd-9378-9f60d363c4b6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:28.922963266Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1928,"client":"1","":"downstairs","session_id":"89bca040-7d33-40cd-9378-9f60d363c4b6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:28.923048143Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1928,"client":"0","":"downstairs","session_id":"89bca040-7d33-40cd-9378-9f60d363c4b6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:28.923281479Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1928,"session_id":"89bca040-7d33-40cd-9378-9f60d363c4b6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:28.923312104Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1928,"session_id":"89bca040-7d33-40cd-9378-9f60d363c4b6"} {"msg":"[0]R flush_numbers[0..12]: [336, 342, 342, 341, 336, 334, 338, 338, 339, 335, 341, 340]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:28.923374775Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1928,"":"downstairs","":"downstairs","session_id":"89bca040-7d33-40cd-9378-9f60d363c4b6"} {"msg":"[0]R generation[0..12]: [192, 196, 196, 196, 192, 190, 194, 194, 194, 191, 196, 195]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:28.923386113Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1928,"":"downstairs","":"downstairs","session_id":"89bca040-7d33-40cd-9378-9f60d363c4b6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:28.923400161Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1928,"":"downstairs","":"downstairs","session_id":"89bca040-7d33-40cd-9378-9f60d363c4b6"} {"msg":"[1]R flush_numbers[0..12]: [336, 342, 342, 341, 336, 334, 338, 338, 339, 335, 341, 340]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:28.923411069Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1928,"":"downstairs","":"downstairs","session_id":"89bca040-7d33-40cd-9378-9f60d363c4b6"} {"msg":"[1]R generation[0..12]: [192, 196, 196, 196, 192, 190, 194, 194, 194, 191, 196, 195]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:28.923421708Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1928,"":"downstairs","":"downstairs","session_id":"89bca040-7d33-40cd-9378-9f60d363c4b6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:28.923432666Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1928,"":"downstairs","":"downstairs","session_id":"89bca040-7d33-40cd-9378-9f60d363c4b6"} {"msg":"[2]R flush_numbers[0..12]: [336, 342, 342, 341, 336, 334, 338, 338, 339, 335, 341, 340]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:28.923443265Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1928,"":"downstairs","":"downstairs","session_id":"89bca040-7d33-40cd-9378-9f60d363c4b6"} {"msg":"[2]R generation[0..12]: [192, 196, 196, 196, 192, 190, 194, 194, 194, 191, 196, 195]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:28.923453783Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1928,"":"downstairs","":"downstairs","session_id":"89bca040-7d33-40cd-9378-9f60d363c4b6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:28.923464632Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1928,"":"downstairs","":"downstairs","session_id":"89bca040-7d33-40cd-9378-9f60d363c4b6"} {"msg":"Max found gen is 198","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:28.92347423Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1928,"":"downstairs","":"downstairs","session_id":"89bca040-7d33-40cd-9378-9f60d363c4b6"} {"msg":"Generation requested: 198 >= found:198","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:28.923484139Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1928,"":"downstairs","":"downstairs","session_id":"89bca040-7d33-40cd-9378-9f60d363c4b6"} {"msg":"Next flush: 345","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:28.923493517Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1928,"":"downstairs","":"downstairs","session_id":"89bca040-7d33-40cd-9378-9f60d363c4b6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:28.923502776Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1928,"":"downstairs","":"downstairs","session_id":"89bca040-7d33-40cd-9378-9f60d363c4b6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:28.923512504Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1928,"session_id":"89bca040-7d33-40cd-9378-9f60d363c4b6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:28.923521293Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1928,"client":"0","":"downstairs","session_id":"89bca040-7d33-40cd-9378-9f60d363c4b6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:28.923531032Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1928,"client":"1","":"downstairs","session_id":"89bca040-7d33-40cd-9378-9f60d363c4b6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:28.92354176Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1928,"client":"2","":"downstairs","session_id":"89bca040-7d33-40cd-9378-9f60d363c4b6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:28.923551779Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1928,"session_id":"89bca040-7d33-40cd-9378-9f60d363c4b6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:28.923560877Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1928,"session_id":"89bca040-7d33-40cd-9378-9f60d363c4b6"} {"msg":"60462765-1256-48a1-b5e6-a2ca8160270b is now active with session: 89bca040-7d33-40cd-9378-9f60d363c4b6","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:28.923570826Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1928,"session_id":"89bca040-7d33-40cd-9378-9f60d363c4b6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:28.923579585Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1928,"session_id":"89bca040-7d33-40cd-9378-9f60d363c4b6"} {"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:28.923588763Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1928} {"msg":"Activated sub_volume 60462765-1256-48a1-b5e6-a2ca8160270b","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:28.923596742Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1928} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:28.923604801Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1928,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:28.923934042Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1928,"task":"crutest"} 01/30 Write block 253 len 28672 data: 19 21 19 21 23 25 22 02/30 Read block 130 len 20480 03/30 Read block 217 len 40960 04/30 Read block 321 len 12288 05/30 Read block 335 len 16384 06/30 Write block 82 len 12288 data: 6 6 8 07/30 Read block 520 len 32768 08/30 Read block 293 len 16384 09/30 Read block 408 len 20480 10/30 Read block 87 len 32768 11/30 Write block 91 len 12288 data: 18 19 18 12/30 Flush 13/30 Write block 216 len 16384 data: 11 9 8 8 14/30 Read block 356 len 32768 15/30 Read block 549 len 20480 16/30 Read block 208 len 36864 17/30 Write block 517 len 40960 data: 20 20 18 17 17 13 10 11 9 9 18/30 Read block 402 len 24576 19/30 Write block 414 len 40960 data: 17 19 20 15 14 13 12 12 15 15 20/30 Write block 152 len 28672 data: 15 16 13 8 11 14 12 21/30 Write block 412 len 20480 data: 18 18 18 20 21 22/30 Read block 73 len 20480 23/30 Write block 506 len 28672 data: 13 12 10 9 15 14 14 24/30 Flush 25/30 Write block 25 len 40960 data: 20 21 18 20 19 17 16 14 11 12 26/30 Read block 353 len 20480 27/30 Read block 399 len 28672 28/30 Read block 275 len 24576 29/30 Write block 575 len 28672 data: 14 13 14 14 12 11 14 30/30 Read block 517 len 20480 ---------------------------------------------------------------- Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Sent Done Done false 1001 Acked Read 5 Sent Done Done false 1002 Acked Read 10 Sent Done Done false 1003 Acked Read 3 Sent Done Done false 1004 Acked Read 4 Sent Done Done false 1005 Acked Write 3 Sent Done Done false 1006 Acked Read 8 Sent Done Done false 1007 Acked Read 4 Sent Done Done false 1008 Acked Read 5 Sent Done Done false 1009 Acked Read 8 Sent Done Done false 1010 Acked Write 3 Sent Done Done false 1011 Acked Flush 0 Sent Done Done false 1012 Acked Write 4 Sent Done Done false 1013 Acked Read 8 Sent Done Done false 1014 Acked Read 5 Sent Done Done false 1015 Acked Read 9 Sent Done Done false 1016 Acked Write 10 Sent Done Done false 1017 Acked Read 6 Sent Done Done false 1018 Acked Write 10 Sent Done Done false 1019 Acked Write 7 Sent Done Done false 1020 Acked Write 5 Sent Done Done false 1021 Acked Read 5 Sent Done Done false 1022 Acked Write 7 Sent Done Done false 1023 Acked Flush 0 Sent Done Done false 1024 Acked Write 10 Sent Done Done false 1025 Acked Read 5 Sent Done Done false 1026 Acked Read 7 Sent Done Done false 1027 Acked Read 6 Sent Done Done false 1028 Acked Write 7 Sent Done Done false 1029 Acked Read 5 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 0 0 30 Done 0 30 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T19:00:28.933231881Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1928} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T19:00:28.933422722Z","hostname":"w-01KYN136ZFZC5JJKNGKY32PPFH","pid":1928,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 192 192 192 336 336 336 F F F 1 020-039 196 196 196 342 342 342 F T T <--- 2 040-059 196 196 196 342 342 342 F F F 3 060-079 196 196 196 341 341 341 F F F 4 080-099 192 198 198 336 345 345 F F F <--- 5 100-119 190 190 190 334 334 334 F F F 6 120-139 194 194 194 338 338 338 F F F 7 140-159 194 198 198 338 346 346 F F F <--- 8 160-179 194 194 194 339 339 339 F F F 9 180-199 191 191 191 335 335 335 F F F 10 200-219 196 198 198 341 346 346 F F F <--- 11 220-239 195 195 195 340 340 340 F F F 12 240-259 196 198 198 343 345 345 F F F <--- 13 260-279 195 195 195 340 340 340 F F F 14 280-299 197 197 197 344 344 344 F F F 15 300-319 197 197 197 344 344 344 F F F 16 320-339 188 188 188 331 331 331 F F F 17 340-359 194 194 194 339 339 339 F F F 18 360-379 193 193 193 337 337 337 F F F 19 380-399 195 195 195 340 340 340 F F F 20 400-419 188 198 198 331 346 346 F F F <--- 21 420-439 192 198 198 336 346 346 F F F <--- 22 440-459 197 197 197 344 344 344 F F F 23 460-479 197 197 197 344 344 344 F F F 24 480-499 197 197 197 344 344 344 F F F 25 500-519 197 198 198 344 346 346 F F F <--- 26 520-539 196 198 198 341 346 346 F F F <--- 27 540-559 196 196 196 341 341 341 F F F 28 560-579 192 192 192 336 336 336 F T T <--- 29 580-599 194 194 194 339 339 339 F T T <--- Max gen: 198, Max flush: 346 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 99 Verifying data now No change after verify Loop: 99 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 192 192 192 336 336 336 F F F 1 020-039 199 199 199 347 347 347 F F F 2 040-059 196 196 196 342 342 342 F F F 3 060-079 196 196 196 341 341 341 F F F 4 080-099 198 198 198 345 345 345 F F F 5 100-119 190 190 190 334 334 334 F F F 6 120-139 194 194 194 338 338 338 F F F 7 140-159 198 198 198 346 346 346 F F F 8 160-179 194 194 194 339 339 339 F F F 9 180-199 191 191 191 335 335 335 F F F 10 200-219 198 198 198 346 346 346 F F F 11 220-239 195 195 195 340 340 340 F F F 12 240-259 198 198 198 345 345 345 F F F 13 260-279 195 195 195 340 340 340 F F F 14 280-299 197 197 197 344 344 344 F F F 15 300-319 197 197 197 344 344 344 F F F 16 320-339 188 188 188 331 331 331 F F F 17 340-359 194 194 194 339 339 339 F F F 18 360-379 193 193 193 337 337 337 F F F 19 380-399 195 195 195 340 340 340 F F F 20 400-419 198 198 198 346 346 346 F F F 21 420-439 198 198 198 346 346 346 F F F 22 440-459 197 197 197 344 344 344 F F F 23 460-479 197 197 197 344 344 344 F F F 24 480-499 197 197 197 344 344 344 F F F 25 500-519 198 198 198 346 346 346 F F F 26 520-539 198 198 198 346 346 346 F F F 27 540-559 196 196 196 341 341 341 F F F 28 560-579 199 199 199 347 347 347 F F F 29 580-599 199 199 199 347 347 347 F F F Max gen: 199, Max flush: 347 7:28 Test duration Test completed real 7:27.667473901 user 25.415534436 sys 14.144929433 trap 0.019524649 tflt 0.067939569 dflt 0.069700134 kflt 0.014420531 lock 11:56:28.691868230 slp 36:15.768220034 lat 2.325119157 stop 0.019360581