Turn off color for downstairs dump Removing existing region "/var/tmp/test_repair" Creating region directory at: /var/tmp/test_repair Creating dsc directory at: /tmp/test_repair/test_repair_dsc Downstairs region 0 created at /var/tmp/test_repair/8810 in 0.20624457 Downstairs region 1 created at /var/tmp/test_repair/8820 in 0.013915349 Downstairs region 2 created at /var/tmp/test_repair/8830 in 0.012642686 Created 3 regions /var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1134 1135 1136 Dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:01.259553894Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:01.272254607Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:01.273366785Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:01.273431323Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:01.273440443Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:01.273451993Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139} {"msg":"Crucible 03fb131c-e7ef-470f-ae70-a1312c53229a has session id: b3b4b86a-be58-42af-9e49-cb934f8220e5","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:01.274098274Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139,"session_id":"b3b4b86a-be58-42af-9e49-cb934f8220e5"} {"msg":"Upstairs opts: Upstairs UUID: 03fb131c-e7ef-470f-ae70-a1312c53229a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:01.27423234Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139,"session_id":"b3b4b86a-be58-42af-9e49-cb934f8220e5"} {"msg":"Crucible stats registered with UUID: 03fb131c-e7ef-470f-ae70-a1312c53229a","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:01.2742431Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139,"session_id":"b3b4b86a-be58-42af-9e49-cb934f8220e5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:42:01.27425253Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139,"session_id":"b3b4b86a-be58-42af-9e49-cb934f8220e5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:01.274750545Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139,"":"io task","client":"0","":"downstairs","session_id":"b3b4b86a-be58-42af-9e49-cb934f8220e5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:01.274792254Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139,"":"io task","client":"1","":"downstairs","session_id":"b3b4b86a-be58-42af-9e49-cb934f8220e5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:01.274804684Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139,"":"io task","client":"2","":"downstairs","session_id":"b3b4b86a-be58-42af-9e49-cb934f8220e5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:01.275058346Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139,"task":"crutest"} {"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:01.275084206Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139} {"msg":"03fb131c-e7ef-470f-ae70-a1312c53229a active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:01.275092825Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139,"session_id":"b3b4b86a-be58-42af-9e49-cb934f8220e5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:01.275101725Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139,"client":"0","":"downstairs","session_id":"b3b4b86a-be58-42af-9e49-cb934f8220e5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:01.275111785Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139,"client":"1","":"downstairs","session_id":"b3b4b86a-be58-42af-9e49-cb934f8220e5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:01.275121434Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139,"client":"2","":"downstairs","session_id":"b3b4b86a-be58-42af-9e49-cb934f8220e5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:01.275131004Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139,"":"io task","client":"2","":"downstairs","session_id":"b3b4b86a-be58-42af-9e49-cb934f8220e5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:01.275141014Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139,"":"io task","client":"0","":"downstairs","session_id":"b3b4b86a-be58-42af-9e49-cb934f8220e5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:01.275183023Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139,"":"io task","client":"1","":"downstairs","session_id":"b3b4b86a-be58-42af-9e49-cb934f8220e5"} {"msg":"ds_connection connected from Ok(127.0.0.1:37832)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:01.275208522Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139,"":"io task","client":"2","":"downstairs","session_id":"b3b4b86a-be58-42af-9e49-cb934f8220e5"} {"msg":"ds_connection connected from Ok(127.0.0.1:46801)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:01.275220792Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139,"":"io task","client":"1","":"downstairs","session_id":"b3b4b86a-be58-42af-9e49-cb934f8220e5"} {"msg":"ds_connection connected from Ok(127.0.0.1:33395)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:01.275426316Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139,"":"io task","client":"0","":"downstairs","session_id":"b3b4b86a-be58-42af-9e49-cb934f8220e5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:01.27701175Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139,"client":"1","":"downstairs","session_id":"b3b4b86a-be58-42af-9e49-cb934f8220e5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:01.277060159Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139,"client":"2","":"downstairs","session_id":"b3b4b86a-be58-42af-9e49-cb934f8220e5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:01.277087628Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139,"client":"0","":"downstairs","session_id":"b3b4b86a-be58-42af-9e49-cb934f8220e5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:01.277566054Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139,"session_id":"b3b4b86a-be58-42af-9e49-cb934f8220e5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:01.277575894Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139,"session_id":"b3b4b86a-be58-42af-9e49-cb934f8220e5"} {"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:01.277683141Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139,"":"downstairs","":"downstairs","session_id":"b3b4b86a-be58-42af-9e49-cb934f8220e5"} {"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:01.27769384Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139,"":"downstairs","":"downstairs","session_id":"b3b4b86a-be58-42af-9e49-cb934f8220e5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:01.27770483Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139,"":"downstairs","":"downstairs","session_id":"b3b4b86a-be58-42af-9e49-cb934f8220e5"} {"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:01.27771484Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139,"":"downstairs","":"downstairs","session_id":"b3b4b86a-be58-42af-9e49-cb934f8220e5"} {"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:01.277724709Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139,"":"downstairs","":"downstairs","session_id":"b3b4b86a-be58-42af-9e49-cb934f8220e5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:01.277735359Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139,"":"downstairs","":"downstairs","session_id":"b3b4b86a-be58-42af-9e49-cb934f8220e5"} {"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:01.277745409Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139,"":"downstairs","":"downstairs","session_id":"b3b4b86a-be58-42af-9e49-cb934f8220e5"} {"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:01.277755368Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139,"":"downstairs","":"downstairs","session_id":"b3b4b86a-be58-42af-9e49-cb934f8220e5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:01.277766078Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139,"":"downstairs","":"downstairs","session_id":"b3b4b86a-be58-42af-9e49-cb934f8220e5"} {"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:01.277775418Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139,"":"downstairs","":"downstairs","session_id":"b3b4b86a-be58-42af-9e49-cb934f8220e5"} {"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:01.277784968Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139,"":"downstairs","":"downstairs","session_id":"b3b4b86a-be58-42af-9e49-cb934f8220e5"} {"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:01.277794127Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139,"":"downstairs","":"downstairs","session_id":"b3b4b86a-be58-42af-9e49-cb934f8220e5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:01.277804867Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139,"":"downstairs","":"downstairs","session_id":"b3b4b86a-be58-42af-9e49-cb934f8220e5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:01.277814387Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139,"session_id":"b3b4b86a-be58-42af-9e49-cb934f8220e5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:01.277823037Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139,"client":"0","":"downstairs","session_id":"b3b4b86a-be58-42af-9e49-cb934f8220e5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:01.277832486Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139,"client":"1","":"downstairs","session_id":"b3b4b86a-be58-42af-9e49-cb934f8220e5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:01.277841926Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139,"client":"2","":"downstairs","session_id":"b3b4b86a-be58-42af-9e49-cb934f8220e5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:01.277851496Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139,"session_id":"b3b4b86a-be58-42af-9e49-cb934f8220e5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:01.277860405Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139,"session_id":"b3b4b86a-be58-42af-9e49-cb934f8220e5"} {"msg":"03fb131c-e7ef-470f-ae70-a1312c53229a is now active with session: b3b4b86a-be58-42af-9e49-cb934f8220e5","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:01.277870105Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139,"session_id":"b3b4b86a-be58-42af-9e49-cb934f8220e5"} {"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:01.277878935Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:01.277886535Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139,"session_id":"b3b4b86a-be58-42af-9e49-cb934f8220e5"} {"msg":"Activated sub_volume 03fb131c-e7ef-470f-ae70-a1312c53229a","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:01.277967522Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:01.278031761Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Fill test","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:01.278224445Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139,"task":"crutest"} Read and Verify all blocks (0..600 range:false) {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30---------------------------------------------------------------- ,"time":"2026-08-03T22:42:01.316465692Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139,"task":"crutest"} {"msg":"CLIENT: Tests done. All submitted work has been ACK'd","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:01.316499311Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139,"task":"crutest"} Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:3 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Read 88 Done Done Done false 1005 Acked Read 256 Done Done Done false 1006 Acked Read 256 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1003)) Some(JobId(1003)) Some(JobId(1003)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1006 1005 1004 1003 1002 {"msg":"CLIENT: Up:0 ds:3 act:3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:01.316565609Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139,"task":"crutest"} ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1007 1006 1005 1004 1003 {"msg":"CLIENT: Up:0 ds:0 act:3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:05.317467955Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139,"task":"crutest"} {"msg":"CLIENT: All crucible jobs finished, exiting program","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:05.317505084Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139,"task":"crutest"} {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:42:05.317514664Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1139} Begin loop 1 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:05.340361884Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1141,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:42:05.347489009Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1141,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:05.347546637Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1141,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:05.354589604Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1141,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:05.354863906Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1141} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:05.354890615Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1141} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:05.354899115Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1141} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:05.354910575Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1141} {"msg":"Crucible 1b7e3f9b-fb49-4d49-95c2-5e215bcb273f has session id: f068c752-f78f-4acc-a5af-3c22cdbd563d","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:05.355267545Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1141,"session_id":"f068c752-f78f-4acc-a5af-3c22cdbd563d"} {"msg":"Upstairs opts: Upstairs UUID: 1b7e3f9b-fb49-4d49-95c2-5e215bcb273f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:05.355324273Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1141,"session_id":"f068c752-f78f-4acc-a5af-3c22cdbd563d"} {"msg":"Crucible stats registered with UUID: 1b7e3f9b-fb49-4d49-95c2-5e215bcb273f","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:05.355336783Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1141,"session_id":"f068c752-f78f-4acc-a5af-3c22cdbd563d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:42:05.355345692Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1141,"session_id":"f068c752-f78f-4acc-a5af-3c22cdbd563d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:05.355485138Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1141,"":"io task","client":"1","":"downstairs","session_id":"f068c752-f78f-4acc-a5af-3c22cdbd563d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:05.355502468Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1141,"":"io task","client":"0","":"downstairs","session_id":"f068c752-f78f-4acc-a5af-3c22cdbd563d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:05.355512467Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1141,"":"io task","client":"2","":"downstairs","session_id":"f068c752-f78f-4acc-a5af-3c22cdbd563d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:05.355666413Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1141,"task":"crutest"} {"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:05.355681723Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1141} {"msg":"1b7e3f9b-fb49-4d49-95c2-5e215bcb273f active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:05.355715172Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1141,"session_id":"f068c752-f78f-4acc-a5af-3c22cdbd563d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:05.355724491Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1141,"client":"0","":"downstairs","session_id":"f068c752-f78f-4acc-a5af-3c22cdbd563d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:05.355734201Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1141,"client":"1","":"downstairs","session_id":"f068c752-f78f-4acc-a5af-3c22cdbd563d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:05.355743921Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1141,"client":"2","":"downstairs","session_id":"f068c752-f78f-4acc-a5af-3c22cdbd563d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:05.355753471Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1141,"":"io task","client":"2","":"downstairs","session_id":"f068c752-f78f-4acc-a5af-3c22cdbd563d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:05.35576342Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1141,"":"io task","client":"0","":"downstairs","session_id":"f068c752-f78f-4acc-a5af-3c22cdbd563d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:05.3557735Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1141,"":"io task","client":"1","":"downstairs","session_id":"f068c752-f78f-4acc-a5af-3c22cdbd563d"} {"msg":"ds_connection connected from Ok(127.0.0.1:44352)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:05.355858587Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1141,"":"io task","client":"1","":"downstairs","session_id":"f068c752-f78f-4acc-a5af-3c22cdbd563d"} {"msg":"ds_connection connected from Ok(127.0.0.1:44049)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:05.355954235Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1141,"":"io task","client":"0","":"downstairs","session_id":"f068c752-f78f-4acc-a5af-3c22cdbd563d"} {"msg":"ds_connection connected from Ok(127.0.0.1:48741)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:05.355968054Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1141,"":"io task","client":"2","":"downstairs","session_id":"f068c752-f78f-4acc-a5af-3c22cdbd563d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:05.356620705Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1141,"client":"2","":"downstairs","session_id":"f068c752-f78f-4acc-a5af-3c22cdbd563d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:05.356658194Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1141,"client":"0","":"downstairs","session_id":"f068c752-f78f-4acc-a5af-3c22cdbd563d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:05.356686734Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1141,"client":"1","":"downstairs","session_id":"f068c752-f78f-4acc-a5af-3c22cdbd563d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:05.356954026Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1141,"session_id":"f068c752-f78f-4acc-a5af-3c22cdbd563d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:05.356963616Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1141,"session_id":"f068c752-f78f-4acc-a5af-3c22cdbd563d"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:05.357002524Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1141,"":"downstairs","":"downstairs","session_id":"f068c752-f78f-4acc-a5af-3c22cdbd563d"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:05.357013194Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1141,"":"downstairs","":"downstairs","session_id":"f068c752-f78f-4acc-a5af-3c22cdbd563d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:05.357024124Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1141,"":"downstairs","":"downstairs","session_id":"f068c752-f78f-4acc-a5af-3c22cdbd563d"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:05.357034234Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1141,"":"downstairs","":"downstairs","session_id":"f068c752-f78f-4acc-a5af-3c22cdbd563d"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:05.357044163Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1141,"":"downstairs","":"downstairs","session_id":"f068c752-f78f-4acc-a5af-3c22cdbd563d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:05.357054953Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1141,"":"downstairs","":"downstairs","session_id":"f068c752-f78f-4acc-a5af-3c22cdbd563d"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:05.357064953Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1141,"":"downstairs","":"downstairs","session_id":"f068c752-f78f-4acc-a5af-3c22cdbd563d"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:05.357074942Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1141,"":"downstairs","":"downstairs","session_id":"f068c752-f78f-4acc-a5af-3c22cdbd563d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:05.357088232Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1141,"":"downstairs","":"downstairs","session_id":"f068c752-f78f-4acc-a5af-3c22cdbd563d"} {"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:05.357097622Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1141,"":"downstairs","":"downstairs","session_id":"f068c752-f78f-4acc-a5af-3c22cdbd563d"} {"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:05.357107161Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1141,"":"downstairs","":"downstairs","session_id":"f068c752-f78f-4acc-a5af-3c22cdbd563d"} {"msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:05.357116301Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1141,"":"downstairs","":"downstairs","session_id":"f068c752-f78f-4acc-a5af-3c22cdbd563d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:05.357125581Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1141,"":"downstairs","":"downstairs","session_id":"f068c752-f78f-4acc-a5af-3c22cdbd563d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:05.357135181Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1141,"session_id":"f068c752-f78f-4acc-a5af-3c22cdbd563d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:05.35714387Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1141,"client":"0","":"downstairs","session_id":"f068c752-f78f-4acc-a5af-3c22cdbd563d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:05.35715371Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1141,"client":"1","":"downstairs","session_id":"f068c752-f78f-4acc-a5af-3c22cdbd563d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:05.3571632Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1141,"client":"2","":"downstairs","session_id":"f068c752-f78f-4acc-a5af-3c22cdbd563d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:05.35717302Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1141,"session_id":"f068c752-f78f-4acc-a5af-3c22cdbd563d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:05.357182189Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1141,"session_id":"f068c752-f78f-4acc-a5af-3c22cdbd563d"} {"msg":"1b7e3f9b-fb49-4d49-95c2-5e215bcb273f is now active with session: f068c752-f78f-4acc-a5af-3c22cdbd563d","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:05.357192029Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1141,"session_id":"f068c752-f78f-4acc-a5af-3c22cdbd563d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:05.357200629Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1141,"session_id":"f068c752-f78f-4acc-a5af-3c22cdbd563d"} {"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:05.357209539Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1141} {"msg":"Activated sub_volume 1b7e3f9b-fb49-4d49-95c2-5e215bcb273f","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:05.357217508Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1141} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:05.357225478Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1141,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:05.357678805Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1141,"task":"crutest"} 01/30 Write block 216 len 8192 data: 2 2 02/30 Read block 384 len 4096 03/30 Write block 285 len 12288 data: 2 2 2 04/30 Read block 455 len 12288 05/30 Read block 235 len 40960 06/30 Read block 140 len 4096 07/30 Flush 08/30 Write block 169 len 12288 data: 2 2 2 09/30 Read block 456 len 8192 10/30 Read block 446 len 16384 11/30 Read block 552 len 28672 12/30 Write block 76 len 32768 data: 2 2 2 2 2 2 2 2 13/30 Write block 576 len 16384 data: 2 2 2 2 14/30 Write block 375 len 32768 data: 2 2 2 2 2 2 2 2 15/30 Read block 438 len 28672 16/30 Write block 411 len 8192 data: 2 2 17/30 Read block 358 len 40960 18/30 Write block 419 len 24576 data: 2 2 2 2 2 2 19/30 Read block 154 len 20480 20/30 Flush 21/30 Write block 557 len 16384 data: 2 2 2 2 22/30 Read block 81 len 8192 23/30 Read block 535 len 40960 24/30 Read block 131 len 16384 25/30 Write block 532 len 16384 data: 2 2 2 2 26/30 Read block 232 len 16384 27/30 Write block 161 len 20480 data: 2 2 2 2 2 28/30 Read block 196 len 12288 29/30 Read block 484 len 4096 30/30 Read block 397 len 28672 ---------------------------------------------------------------- Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1007 Acked Write 3 Sent Done Done false 1008 Acked Read 2 Sent Done Done false 1009 Acked Read 4 Sent Done Done false 1010 Acked Read 7 Sent Done Done false 1011 Acked Write 8 Sent Done Done false 1012 Acked Write 4 Sent Done Done false 1013 Acked Write 8 Sent Done Done false 1014 Acked Read 7 Sent Done Done false 1015 Acked Write 2 Sent Done Done false 1016 Acked Read 10 Sent Done Done false 1017 Acked Write 6 Sent Done Done false 1018 Acked Read 5 Sent Done Done false 1019 Acked Flush 0 Sent Done Done false 1020 Acked Write 4 Sent Done Done false 1021 Acked Read 2 Sent Done Done false 1022 Acked Read 10 Sent Done Done false 1023 Acked Read 4 Sent Done Done false 1024 Acked Write 4 Sent Done Done false 1025 Acked Read 4 Sent Done Done false 1026 Acked Write 5 Sent Done Done false 1027 Acked Read 3 Sent Done Done false 1028 Acked Read 1 Sent Done Done false 1029 Acked Read 7 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 23 1 0 24 Done 0 22 23 45 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1006)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:42:05.366730464Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1141} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:05.366905739Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1141,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 1 1 1 1 1 1 F F F 3 060-079 1 2 2 1 3 3 F F F <--- 4 080-099 1 2 2 1 3 3 F F F <--- 5 100-119 1 1 1 1 1 1 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 1 2 2 1 3 3 F T T <--- 9 180-199 1 1 1 1 1 1 F F F 10 200-219 2 2 2 2 2 2 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 2 2 2 2 2 2 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 1 2 2 1 3 3 F F F <--- 19 380-399 1 2 2 1 3 3 F F F <--- 20 400-419 1 2 2 1 3 3 F F F <--- 21 420-439 1 2 2 1 3 3 F F F <--- 22 440-459 1 1 1 1 1 1 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 1 1 1 1 1 1 F T T <--- 27 540-559 1 1 1 1 1 1 F T T <--- 28 560-579 1 2 2 1 3 3 F T T <--- 29 580-599 1 1 1 1 1 1 F F F Max gen: 2, Max flush: 3 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 1 Verifying data now No change after verify Loop: 1 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 1 1 1 1 1 1 F F F 3 060-079 2 2 2 3 3 3 F F F 4 080-099 2 2 2 3 3 3 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 3 3 3 4 4 4 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 2 2 2 2 2 2 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 2 2 2 2 2 2 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 2 2 2 3 3 3 F F F 19 380-399 2 2 2 3 3 3 F F F 20 400-419 2 2 2 3 3 3 F F F 21 420-439 2 2 2 3 3 3 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 3 3 3 4 4 4 F F F 27 540-559 3 3 3 4 4 4 F F F 28 560-579 3 3 3 4 4 4 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 3, Max flush: 4 Begin loop 2 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:09.732359379Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1149,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:09.740530693Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1149,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:09.740815005Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1149} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:09.740839464Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1149} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:09.740847754Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1149} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:09.740859164Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1149} {"msg":"Crucible 1736d776-f251-4db7-89cb-935429bc5bb6 has session id: 799407b7-62a7-4598-a01b-341a6bf02c60","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:09.741171545Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1149,"session_id":"799407b7-62a7-4598-a01b-341a6bf02c60"} {"msg":"Upstairs opts: Upstairs UUID: 1736d776-f251-4db7-89cb-935429bc5bb6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:09.741203334Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1149,"session_id":"799407b7-62a7-4598-a01b-341a6bf02c60"} {"msg":"Crucible stats registered with UUID: 1736d776-f251-4db7-89cb-935429bc5bb6","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:09.741228193Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1149,"session_id":"799407b7-62a7-4598-a01b-341a6bf02c60"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:42:09.741238033Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1149,"session_id":"799407b7-62a7-4598-a01b-341a6bf02c60"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:09.74134441Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1149,"":"io task","client":"2","":"downstairs","session_id":"799407b7-62a7-4598-a01b-341a6bf02c60"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:09.741374369Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1149,"":"io task","client":"0","":"downstairs","session_id":"799407b7-62a7-4598-a01b-341a6bf02c60"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:09.741385218Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1149,"":"io task","client":"1","":"downstairs","session_id":"799407b7-62a7-4598-a01b-341a6bf02c60"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:09.741525074Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1149,"task":"crutest"} {"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:09.741542524Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1149} {"msg":"1736d776-f251-4db7-89cb-935429bc5bb6 active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:09.741558663Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1149,"session_id":"799407b7-62a7-4598-a01b-341a6bf02c60"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:09.741569163Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1149,"client":"0","":"downstairs","session_id":"799407b7-62a7-4598-a01b-341a6bf02c60"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:09.741578823Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1149,"client":"1","":"downstairs","session_id":"799407b7-62a7-4598-a01b-341a6bf02c60"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:09.741588443Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1149,"client":"2","":"downstairs","session_id":"799407b7-62a7-4598-a01b-341a6bf02c60"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:09.741597952Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1149,"":"io task","client":"2","":"downstairs","session_id":"799407b7-62a7-4598-a01b-341a6bf02c60"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:09.741607882Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1149,"":"io task","client":"0","":"downstairs","session_id":"799407b7-62a7-4598-a01b-341a6bf02c60"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:09.741617672Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1149,"":"io task","client":"1","":"downstairs","session_id":"799407b7-62a7-4598-a01b-341a6bf02c60"} {"msg":"ds_connection connected from Ok(127.0.0.1:62920)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:09.741700029Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1149,"":"io task","client":"2","":"downstairs","session_id":"799407b7-62a7-4598-a01b-341a6bf02c60"} {"msg":"ds_connection connected from Ok(127.0.0.1:38927)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:09.741713729Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1149,"":"io task","client":"1","":"downstairs","session_id":"799407b7-62a7-4598-a01b-341a6bf02c60"} {"msg":"ds_connection connected from Ok(127.0.0.1:33186)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:09.741838075Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1149,"":"io task","client":"0","":"downstairs","session_id":"799407b7-62a7-4598-a01b-341a6bf02c60"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:09.74237342Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1149,"client":"2","":"downstairs","session_id":"799407b7-62a7-4598-a01b-341a6bf02c60"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:09.742390419Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1149,"client":"0","":"downstairs","session_id":"799407b7-62a7-4598-a01b-341a6bf02c60"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:09.742452808Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1149,"client":"1","":"downstairs","session_id":"799407b7-62a7-4598-a01b-341a6bf02c60"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:09.742689181Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1149,"session_id":"799407b7-62a7-4598-a01b-341a6bf02c60"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:09.74272425Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1149,"session_id":"799407b7-62a7-4598-a01b-341a6bf02c60"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 3, 3, 1, 1, 1, 4, 1, 2, 1]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:09.742788258Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1149,"":"downstairs","":"downstairs","session_id":"799407b7-62a7-4598-a01b-341a6bf02c60"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 2, 2, 1, 1, 1, 3, 1, 2, 1]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:09.742801978Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1149,"":"downstairs","":"downstairs","session_id":"799407b7-62a7-4598-a01b-341a6bf02c60"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:09.742812937Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1149,"":"downstairs","":"downstairs","session_id":"799407b7-62a7-4598-a01b-341a6bf02c60"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 3, 3, 1, 1, 1, 4, 1, 2, 1]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:09.742823167Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1149,"":"downstairs","":"downstairs","session_id":"799407b7-62a7-4598-a01b-341a6bf02c60"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 2, 2, 1, 1, 1, 3, 1, 2, 1]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:09.742833197Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1149,"":"downstairs","":"downstairs","session_id":"799407b7-62a7-4598-a01b-341a6bf02c60"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:09.742843906Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1149,"":"downstairs","":"downstairs","session_id":"799407b7-62a7-4598-a01b-341a6bf02c60"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 3, 3, 1, 1, 1, 4, 1, 2, 1]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:09.742854086Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1149,"":"downstairs","":"downstairs","session_id":"799407b7-62a7-4598-a01b-341a6bf02c60"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 2, 2, 1, 1, 1, 3, 1, 2, 1]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:09.742864196Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1149,"":"downstairs","":"downstairs","session_id":"799407b7-62a7-4598-a01b-341a6bf02c60"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:09.742874905Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1149,"":"downstairs","":"downstairs","session_id":"799407b7-62a7-4598-a01b-341a6bf02c60"} {"msg":"Max found gen is 4","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:09.742884175Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1149,"":"downstairs","":"downstairs","session_id":"799407b7-62a7-4598-a01b-341a6bf02c60"} {"msg":"Generation requested: 4 >= found:4","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:09.742893995Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1149,"":"downstairs","":"downstairs","session_id":"799407b7-62a7-4598-a01b-341a6bf02c60"} {"msg":"Next flush: 5","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:09.742903105Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1149,"":"downstairs","":"downstairs","session_id":"799407b7-62a7-4598-a01b-341a6bf02c60"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:09.742912374Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1149,"":"downstairs","":"downstairs","session_id":"799407b7-62a7-4598-a01b-341a6bf02c60"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:09.742922234Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1149,"session_id":"799407b7-62a7-4598-a01b-341a6bf02c60"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:09.742930994Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1149,"client":"0","":"downstairs","session_id":"799407b7-62a7-4598-a01b-341a6bf02c60"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:09.742940974Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1149,"client":"1","":"downstairs","session_id":"799407b7-62a7-4598-a01b-341a6bf02c60"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:09.742952163Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1149,"client":"2","":"downstairs","session_id":"799407b7-62a7-4598-a01b-341a6bf02c60"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:09.742962113Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1149,"session_id":"799407b7-62a7-4598-a01b-341a6bf02c60"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:09.742971353Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1149,"session_id":"799407b7-62a7-4598-a01b-341a6bf02c60"} {"msg":"1736d776-f251-4db7-89cb-935429bc5bb6 is now active with session: 799407b7-62a7-4598-a01b-341a6bf02c60","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:09.742981452Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1149,"session_id":"799407b7-62a7-4598-a01b-341a6bf02c60"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:09.742990172Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1149,"session_id":"799407b7-62a7-4598-a01b-341a6bf02c60"} {"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:09.742999152Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1149} {"msg":"Activated sub_volume 1736d776-f251-4db7-89cb-935429bc5bb6","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:09.743007122Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1149} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:09.743015061Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1149,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:09.743298923Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1149,"task":"crutest"} 01/30 Write block 364 len 4096 data: 2 02/30 Read block 16 len 4096 03/30 Write block 289 len 24576 data: 2 2 2 2 2 2 04/30 Write block 352 len 28672 data: 2 2 2 2 2 2 2 05/30 Read block 25 len 36864 06/30 Read block 118 len 36864 07/30 Read block 196 len 24576 08/30 Read block 63 len 36864 09/30 Flush 10/30 Write block 134 len 4096 data: 2 11/30 Write block 253 len 20480 data: 2 2 2 2 2 12/30 Read block 474 len 4096 13/30 Read block 307 len 4096 14/30 Write block 212 len 24576 data: 2 2 2 2 3 3 15/30 Write block 331 len 28672 data: 2 2 2 2 2 2 2 16/30 Read block 112 len 16384 17/30 Write block 548 len 12288 data: 2 2 2 18/30 Read block 559 len 24576 19/30 Write block 328 len 16384 data: 2 2 2 3 20/30 Read block 215 len 4096 21/30 Write block 126 len 32768 data: 2 2 2 2 2 2 2 2 22/30 Read block 107 len 40960 23/30 Write block 450 len 8192 data: 2 2 24/30 Read block 105 len 40960 25/30 Write block 521 len 20480 data: 2 2 2 2 2 26/30 Read block 491 len 12288 27/30 Write block 248 len 12288 data: 2 2 2 28/30 Read block 30 len 16384 29/30 Write block 58 len 4096 data: 2 30/30 Write block 191 len 36864 data: 2 2 2 2 2 2 2 2 2 ---------------------------------------------------------------- Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Read 1 Done Done Done false 1002 Acked Write 6 Done Done Done false 1003 Acked Write 7 Done Sent Done false 1004 Acked Read 9 Done Sent Done false 1005 Acked Read 9 Done Sent Done false 1006 Acked Read 6 Done Sent Done false 1007 Acked Read 9 Done Sent Done false 1008 Acked Flush 0 Done Sent Done false 1009 Acked Write 1 Done Sent Done false 1010 Acked Write 5 Done Sent Done false 1011 Acked Read 1 Done Sent Done false 1012 Acked Read 1 Done Sent Done false 1013 Acked Write 6 Done Sent Done false 1014 Acked Write 7 Done Sent Done false 1015 Acked Read 4 Done Sent Done false 1016 Acked Write 3 Done Sent Done false 1017 Acked Read 6 Done Sent Done false 1018 Acked Write 4 Done Sent Done false 1019 Acked Read 1 Done Sent Done false 1020 Acked Write 8 Done Sent Done false 1021 Acked Read 10 Done Sent Done false 1022 Acked Write 2 Done Sent Done false 1023 Acked Read 10 Done Sent Done false 1024 Acked Write 5 Done Sent Done false 1025 Acked Read 3 Done Sent Done false 1026 Acked Write 3 Done Sent Done false 1027 Acked Read 4 Done Sent Done false 1028 Acked Write 1 Done Sent Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 27 2 30 Done 29 3 28 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1008)) None Some(JobId(1008)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:42:09.751151817Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1149} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:09.751323682Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1149,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 1 1 1 1 1 1 T F T <--- 3 060-079 2 2 2 3 3 3 F F F 4 080-099 2 2 2 3 3 3 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 1 1 1 1 1 1 T F T <--- 7 140-159 1 1 1 1 1 1 F F F 8 160-179 3 3 3 4 4 4 F F F 9 180-199 1 1 1 1 1 1 T F T <--- 10 200-219 2 2 2 2 2 2 T F T <--- 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 T F T <--- 13 260-279 1 1 1 1 1 1 F F F 14 280-299 4 2 4 5 2 5 F T F <--- 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 T F T <--- 17 340-359 4 1 4 5 1 5 F F F <--- 18 360-379 4 2 4 5 3 5 F T F <--- 19 380-399 2 2 2 3 3 3 F F F 20 400-419 2 2 2 3 3 3 F F F 21 420-439 2 2 2 3 3 3 F F F 22 440-459 1 1 1 1 1 1 T F T <--- 23 460-479 1 1 1 1 1 1 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 3 3 3 4 4 4 T F T <--- 27 540-559 3 3 3 4 4 4 T F T <--- 28 560-579 3 3 3 4 4 4 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 4, Max flush: 5 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 2 Verifying data now No change after verify Loop: 2 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 5 5 5 6 6 6 F F F 3 060-079 2 2 2 3 3 3 F F F 4 080-099 2 2 2 3 3 3 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 5 5 5 6 6 6 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 3 3 3 4 4 4 F F F 9 180-199 5 5 5 6 6 6 F F F 10 200-219 5 5 5 6 6 6 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 5 5 5 6 6 6 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 4 4 4 5 5 5 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 5 5 5 6 6 6 F F F 17 340-359 4 4 4 5 5 5 F F F 18 360-379 4 4 4 5 5 5 F F F 19 380-399 2 2 2 3 3 3 F F F 20 400-419 2 2 2 3 3 3 F F F 21 420-439 2 2 2 3 3 3 F F F 22 440-459 5 5 5 6 6 6 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 5 5 5 6 6 6 F F F 27 540-559 5 5 5 6 6 6 F F F 28 560-579 3 3 3 4 4 4 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 5, Max flush: 6 Begin loop 3 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:13.992933124Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1157,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:14.000877394Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1157,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:14.001193475Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1157} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:14.001221175Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1157} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:14.001230124Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1157} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:14.001241904Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1157} {"msg":"Crucible 406c5ca7-cec9-4a9e-8419-420bfcb48f68 has session id: 48bc8a01-d5a4-45ca-9e5e-d1f5e143fe65","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:14.001623403Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1157,"session_id":"48bc8a01-d5a4-45ca-9e5e-d1f5e143fe65"} {"msg":"Upstairs opts: Upstairs UUID: 406c5ca7-cec9-4a9e-8419-420bfcb48f68, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:14.001659532Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1157,"session_id":"48bc8a01-d5a4-45ca-9e5e-d1f5e143fe65"} {"msg":"Crucible stats registered with UUID: 406c5ca7-cec9-4a9e-8419-420bfcb48f68","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:14.001670932Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1157,"session_id":"48bc8a01-d5a4-45ca-9e5e-d1f5e143fe65"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:42:14.001680141Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1157,"session_id":"48bc8a01-d5a4-45ca-9e5e-d1f5e143fe65"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:14.001836017Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1157,"":"io task","client":"1","":"downstairs","session_id":"48bc8a01-d5a4-45ca-9e5e-d1f5e143fe65"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:14.001871406Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1157,"":"io task","client":"0","":"downstairs","session_id":"48bc8a01-d5a4-45ca-9e5e-d1f5e143fe65"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:14.001882315Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1157,"":"io task","client":"2","":"downstairs","session_id":"48bc8a01-d5a4-45ca-9e5e-d1f5e143fe65"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:14.001976773Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1157,"task":"crutest"} {"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:14.001992832Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1157} {"msg":"406c5ca7-cec9-4a9e-8419-420bfcb48f68 active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:14.002023421Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1157,"session_id":"48bc8a01-d5a4-45ca-9e5e-d1f5e143fe65"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:14.002048981Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1157,"client":"0","":"downstairs","session_id":"48bc8a01-d5a4-45ca-9e5e-d1f5e143fe65"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:14.00205995Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1157,"client":"1","":"downstairs","session_id":"48bc8a01-d5a4-45ca-9e5e-d1f5e143fe65"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:14.00206982Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1157,"client":"2","":"downstairs","session_id":"48bc8a01-d5a4-45ca-9e5e-d1f5e143fe65"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:14.00207947Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1157,"":"io task","client":"2","":"downstairs","session_id":"48bc8a01-d5a4-45ca-9e5e-d1f5e143fe65"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:14.002089509Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1157,"":"io task","client":"0","":"downstairs","session_id":"48bc8a01-d5a4-45ca-9e5e-d1f5e143fe65"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:14.002099309Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1157,"":"io task","client":"1","":"downstairs","session_id":"48bc8a01-d5a4-45ca-9e5e-d1f5e143fe65"} {"msg":"ds_connection connected from Ok(127.0.0.1:33466)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:14.002171367Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1157,"":"io task","client":"0","":"downstairs","session_id":"48bc8a01-d5a4-45ca-9e5e-d1f5e143fe65"} {"msg":"ds_connection connected from Ok(127.0.0.1:41654)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:14.002187237Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1157,"":"io task","client":"1","":"downstairs","session_id":"48bc8a01-d5a4-45ca-9e5e-d1f5e143fe65"} {"msg":"ds_connection connected from Ok(127.0.0.1:63509)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:14.002197886Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1157,"":"io task","client":"2","":"downstairs","session_id":"48bc8a01-d5a4-45ca-9e5e-d1f5e143fe65"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:14.002914636Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1157,"client":"1","":"downstairs","session_id":"48bc8a01-d5a4-45ca-9e5e-d1f5e143fe65"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:14.002939985Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1157,"client":"0","":"downstairs","session_id":"48bc8a01-d5a4-45ca-9e5e-d1f5e143fe65"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:14.003032702Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1157,"client":"2","":"downstairs","session_id":"48bc8a01-d5a4-45ca-9e5e-d1f5e143fe65"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:14.003230627Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1157,"session_id":"48bc8a01-d5a4-45ca-9e5e-d1f5e143fe65"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:14.003247406Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1157,"session_id":"48bc8a01-d5a4-45ca-9e5e-d1f5e143fe65"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 6, 3, 3, 1, 6, 1, 4, 6, 6, 1]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:14.003335634Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1157,"":"downstairs","":"downstairs","session_id":"48bc8a01-d5a4-45ca-9e5e-d1f5e143fe65"} {"msg":"[0]R generation[0..12]: [1, 1, 5, 2, 2, 1, 5, 1, 3, 5, 5, 1]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:14.003346573Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1157,"":"downstairs","":"downstairs","session_id":"48bc8a01-d5a4-45ca-9e5e-d1f5e143fe65"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:14.003357593Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1157,"":"downstairs","":"downstairs","session_id":"48bc8a01-d5a4-45ca-9e5e-d1f5e143fe65"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 6, 3, 3, 1, 6, 1, 4, 6, 6, 1]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:14.003367813Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1157,"":"downstairs","":"downstairs","session_id":"48bc8a01-d5a4-45ca-9e5e-d1f5e143fe65"} {"msg":"[1]R generation[0..12]: [1, 1, 5, 2, 2, 1, 5, 1, 3, 5, 5, 1]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:14.003377852Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1157,"":"downstairs","":"downstairs","session_id":"48bc8a01-d5a4-45ca-9e5e-d1f5e143fe65"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:14.003388732Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1157,"":"downstairs","":"downstairs","session_id":"48bc8a01-d5a4-45ca-9e5e-d1f5e143fe65"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 6, 3, 3, 1, 6, 1, 4, 6, 6, 1]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:14.003398812Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1157,"":"downstairs","":"downstairs","session_id":"48bc8a01-d5a4-45ca-9e5e-d1f5e143fe65"} {"msg":"[2]R generation[0..12]: [1, 1, 5, 2, 2, 1, 5, 1, 3, 5, 5, 1]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:14.003410971Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1157,"":"downstairs","":"downstairs","session_id":"48bc8a01-d5a4-45ca-9e5e-d1f5e143fe65"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:14.003421881Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1157,"":"downstairs","":"downstairs","session_id":"48bc8a01-d5a4-45ca-9e5e-d1f5e143fe65"} {"msg":"Max found gen is 6","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:14.003431411Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1157,"":"downstairs","":"downstairs","session_id":"48bc8a01-d5a4-45ca-9e5e-d1f5e143fe65"} {"msg":"Generation requested: 6 >= found:6","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:14.003441211Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1157,"":"downstairs","":"downstairs","session_id":"48bc8a01-d5a4-45ca-9e5e-d1f5e143fe65"} {"msg":"Next flush: 7","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:14.00345056Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1157,"":"downstairs","":"downstairs","session_id":"48bc8a01-d5a4-45ca-9e5e-d1f5e143fe65"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:14.0034603Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1157,"":"downstairs","":"downstairs","session_id":"48bc8a01-d5a4-45ca-9e5e-d1f5e143fe65"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:14.00347064Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1157,"session_id":"48bc8a01-d5a4-45ca-9e5e-d1f5e143fe65"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:14.003479479Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1157,"client":"0","":"downstairs","session_id":"48bc8a01-d5a4-45ca-9e5e-d1f5e143fe65"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:14.003489229Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1157,"client":"1","":"downstairs","session_id":"48bc8a01-d5a4-45ca-9e5e-d1f5e143fe65"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:14.003498969Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1157,"client":"2","":"downstairs","session_id":"48bc8a01-d5a4-45ca-9e5e-d1f5e143fe65"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:14.003508849Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1157,"session_id":"48bc8a01-d5a4-45ca-9e5e-d1f5e143fe65"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:14.003517868Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1157,"session_id":"48bc8a01-d5a4-45ca-9e5e-d1f5e143fe65"} {"msg":"406c5ca7-cec9-4a9e-8419-420bfcb48f68 is now active with session: 48bc8a01-d5a4-45ca-9e5e-d1f5e143fe65","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:14.003527798Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1157,"session_id":"48bc8a01-d5a4-45ca-9e5e-d1f5e143fe65"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:14.003536588Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1157,"session_id":"48bc8a01-d5a4-45ca-9e5e-d1f5e143fe65"} {"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:14.003545587Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1157} {"msg":"Activated sub_volume 406c5ca7-cec9-4a9e-8419-420bfcb48f68","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:14.003553587Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1157} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:14.003561647Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1157,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:14.003853539Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1157,"task":"crutest"} 01/30 Flush 02/30 Write block 582 len 40960 data: 2 2 2 2 2 2 2 2 2 2 03/30 Flush 04/30 Write block 18 len 12288 data: 2 2 2 05/30 Write block 397 len 32768 data: 2 2 2 2 2 2 2 2 06/30 Write block 321 len 40960 data: 2 2 2 2 2 2 2 3 3 3 07/30 Write block 576 len 28672 data: 3 3 3 3 2 2 3 08/30 Read block 100 len 16384 09/30 Read block 349 len 24576 10/30 Write block 361 len 28672 data: 2 2 2 3 2 2 2 11/30 Write block 316 len 4096 data: 2 12/30 Write block 435 len 36864 data: 2 2 2 2 2 2 2 2 2 13/30 Flush 14/30 Write block 223 len 32768 data: 2 2 2 2 2 2 2 2 15/30 Flush 16/30 Write block 485 len 12288 data: 2 2 2 17/30 Read block 351 len 12288 18/30 Write block 337 len 28672 data: 3 2 2 2 2 2 2 19/30 Read block 176 len 36864 20/30 Read block 573 len 24576 21/30 Write block 210 len 12288 data: 2 2 3 22/30 Flush 23/30 Write block 511 len 12288 data: 2 2 2 24/30 Read block 27 len 4096 25/30 Write block 583 len 40960 data: 3 3 3 3 3 3 3 3 3 2 26/30 Write block 432 len 24576 data: 2 2 2 3 3 3 27/30 Read block 202 len 8192 28/30 Read block 92 len 8192 29/30 Write block 184 len 16384 data: 2 2 2 2 30/30 Read block 472 len 16384 ---------------------------------------------------------------- Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Flush 0 Done Done Sent false 1001 Acked Write 10 Done Done Sent false 1002 Acked Flush 0 Done Done Sent false 1003 Acked Write 3 Done Done Sent false 1004 Acked Write 8 Done Done Sent false 1005 Acked Write 10 Done Done Sent false 1006 Acked Write 7 Done Done Sent false 1007 Acked Read 4 Done Done Sent false 1008 Acked Read 6 Done Done Sent false 1009 Acked Write 7 Done Done Sent false 1010 Acked Write 1 Done Done Sent false 1011 Acked Write 9 Done Done Sent false 1012 Acked Flush 0 Done Done Sent false 1013 Acked Write 8 Done Done Sent false 1014 Acked Flush 0 Done Done Sent false 1015 Acked Write 3 Done Done Sent false 1016 Acked Read 3 Done Done Sent false 1017 Acked Write 7 Done Done Sent false 1018 Acked Read 9 Done Done Sent false 1019 Acked Read 6 Done Done Sent false 1020 Acked Write 3 Done Done Sent false 1021 Acked Flush 0 Done Done Sent false 1022 Acked Write 3 Done Done Sent false 1023 Acked Read 1 Done Done Sent false 1024 Acked Write 10 Done Done Sent false 1025 Acked Write 6 Done Done Sent false 1026 Acked Read 2 Done Done Sent false 1027 Acked Read 2 Done Done Sent false 1028 Acked Write 4 Done Done Sent false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 30 31 Done 30 29 0 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:42:14.012081711Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1157} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:14.012252836Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1157,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 6 6 1 9 9 1 F F F <--- 1 020-039 6 6 1 9 9 1 F F F <--- 2 040-059 5 5 5 6 6 6 F F F 3 060-079 2 2 2 3 3 3 F F F 4 080-099 2 2 2 3 3 3 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 5 5 5 6 6 6 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 3 3 3 4 4 4 F F F 9 180-199 5 5 5 6 6 6 T T F <--- 10 200-219 6 6 5 11 11 6 F F F <--- 11 220-239 6 6 1 10 10 1 F F F <--- 12 240-259 5 5 5 6 6 6 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 4 4 4 5 5 5 F F F 15 300-319 6 6 1 9 9 1 F F F <--- 16 320-339 6 6 5 11 11 6 F F F <--- 17 340-359 6 6 4 11 11 5 F F F <--- 18 360-379 6 6 4 9 9 5 F F F <--- 19 380-399 6 6 2 9 9 3 F F F <--- 20 400-419 6 6 2 9 9 3 F F F <--- 21 420-439 6 6 2 9 9 3 T T F <--- 22 440-459 6 6 5 9 9 6 F F F <--- 23 460-479 1 1 1 1 1 1 F F F 24 480-499 6 6 1 11 11 1 F F F <--- 25 500-519 1 1 1 1 1 1 T T F <--- 26 520-539 5 5 5 6 6 6 F F F 27 540-559 5 5 5 6 6 6 F F F 28 560-579 6 6 3 9 9 4 F F F <--- 29 580-599 6 6 1 9 9 1 T T F <--- Max gen: 6, Max flush: 11 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 3 Verifying data now No change after verify Loop: 3 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 6 6 6 9 9 9 F F F 1 020-039 6 6 6 9 9 9 F F F 2 040-059 5 5 5 6 6 6 F F F 3 060-079 2 2 2 3 3 3 F F F 4 080-099 2 2 2 3 3 3 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 5 5 5 6 6 6 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 3 3 3 4 4 4 F F F 9 180-199 7 7 7 12 12 12 F F F 10 200-219 6 6 6 11 11 11 F F F 11 220-239 6 6 6 10 10 10 F F F 12 240-259 5 5 5 6 6 6 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 4 4 4 5 5 5 F F F 15 300-319 6 6 6 9 9 9 F F F 16 320-339 6 6 6 11 11 11 F F F 17 340-359 6 6 6 11 11 11 F F F 18 360-379 6 6 6 9 9 9 F F F 19 380-399 6 6 6 9 9 9 F F F 20 400-419 6 6 6 9 9 9 F F F 21 420-439 7 7 7 12 12 12 F F F 22 440-459 6 6 6 9 9 9 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 6 6 6 11 11 11 F F F 25 500-519 7 7 7 12 12 12 F F F 26 520-539 5 5 5 6 6 6 F F F 27 540-559 5 5 5 6 6 6 F F F 28 560-579 6 6 6 9 9 9 F F F 29 580-599 7 7 7 12 12 12 F F F Max gen: 7, Max flush: 12 Begin loop 4 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:18.25866098Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1165,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:42:18.265633219Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1165,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:18.265688348Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1165,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:18.272490591Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1165,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:18.272752134Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1165} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:18.272778693Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1165} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:18.272787203Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1165} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:18.272798652Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1165} {"msg":"Crucible ef93920d-7f2c-45b7-8343-61f2ad18540a has session id: 477622a3-9fac-45be-b783-6a65254e0141","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:18.273159122Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1165,"session_id":"477622a3-9fac-45be-b783-6a65254e0141"} {"msg":"Upstairs opts: Upstairs UUID: ef93920d-7f2c-45b7-8343-61f2ad18540a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:18.273178302Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1165,"session_id":"477622a3-9fac-45be-b783-6a65254e0141"} {"msg":"Crucible stats registered with UUID: ef93920d-7f2c-45b7-8343-61f2ad18540a","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:18.273188131Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1165,"session_id":"477622a3-9fac-45be-b783-6a65254e0141"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:42:18.273197091Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1165,"session_id":"477622a3-9fac-45be-b783-6a65254e0141"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:18.273334467Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1165,"":"io task","client":"1","":"downstairs","session_id":"477622a3-9fac-45be-b783-6a65254e0141"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:18.273353276Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1165,"":"io task","client":"0","":"downstairs","session_id":"477622a3-9fac-45be-b783-6a65254e0141"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:18.273363326Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1165,"":"io task","client":"2","":"downstairs","session_id":"477622a3-9fac-45be-b783-6a65254e0141"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:18.273498132Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1165,"task":"crutest"} {"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:18.273521952Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1165} {"msg":"ef93920d-7f2c-45b7-8343-61f2ad18540a active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:18.273554941Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1165,"session_id":"477622a3-9fac-45be-b783-6a65254e0141"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:18.27356432Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1165,"client":"0","":"downstairs","session_id":"477622a3-9fac-45be-b783-6a65254e0141"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:18.27357423Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1165,"client":"1","":"downstairs","session_id":"477622a3-9fac-45be-b783-6a65254e0141"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:18.27358385Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1165,"client":"2","":"downstairs","session_id":"477622a3-9fac-45be-b783-6a65254e0141"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:18.27359345Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1165,"":"io task","client":"2","":"downstairs","session_id":"477622a3-9fac-45be-b783-6a65254e0141"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:18.273603349Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1165,"":"io task","client":"0","":"downstairs","session_id":"477622a3-9fac-45be-b783-6a65254e0141"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:18.273613249Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1165,"":"io task","client":"1","":"downstairs","session_id":"477622a3-9fac-45be-b783-6a65254e0141"} {"msg":"ds_connection connected from Ok(127.0.0.1:34963)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:18.273706146Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1165,"":"io task","client":"1","":"downstairs","session_id":"477622a3-9fac-45be-b783-6a65254e0141"} {"msg":"ds_connection connected from Ok(127.0.0.1:63907)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:18.273731306Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1165,"":"io task","client":"0","":"downstairs","session_id":"477622a3-9fac-45be-b783-6a65254e0141"} {"msg":"ds_connection connected from Ok(127.0.0.1:36319)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:18.273742615Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1165,"":"io task","client":"2","":"downstairs","session_id":"477622a3-9fac-45be-b783-6a65254e0141"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:18.274441815Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1165,"client":"2","":"downstairs","session_id":"477622a3-9fac-45be-b783-6a65254e0141"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:18.274495444Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1165,"client":"0","":"downstairs","session_id":"477622a3-9fac-45be-b783-6a65254e0141"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:18.274513823Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1165,"client":"1","":"downstairs","session_id":"477622a3-9fac-45be-b783-6a65254e0141"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:18.274790075Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1165,"session_id":"477622a3-9fac-45be-b783-6a65254e0141"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:18.274825364Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1165,"session_id":"477622a3-9fac-45be-b783-6a65254e0141"} {"msg":"[0]R flush_numbers[0..12]: [9, 9, 6, 3, 3, 1, 6, 1, 4, 12, 11, 10]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:18.274878782Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1165,"":"downstairs","":"downstairs","session_id":"477622a3-9fac-45be-b783-6a65254e0141"} {"msg":"[0]R generation[0..12]: [6, 6, 5, 2, 2, 1, 5, 1, 3, 7, 6, 6]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:18.274889552Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1165,"":"downstairs","":"downstairs","session_id":"477622a3-9fac-45be-b783-6a65254e0141"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:18.274900462Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1165,"":"downstairs","":"downstairs","session_id":"477622a3-9fac-45be-b783-6a65254e0141"} {"msg":"[1]R flush_numbers[0..12]: [9, 9, 6, 3, 3, 1, 6, 1, 4, 12, 11, 10]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:18.274910742Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1165,"":"downstairs","":"downstairs","session_id":"477622a3-9fac-45be-b783-6a65254e0141"} {"msg":"[1]R generation[0..12]: [6, 6, 5, 2, 2, 1, 5, 1, 3, 7, 6, 6]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:18.274920871Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1165,"":"downstairs","":"downstairs","session_id":"477622a3-9fac-45be-b783-6a65254e0141"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:18.274931601Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1165,"":"downstairs","":"downstairs","session_id":"477622a3-9fac-45be-b783-6a65254e0141"} {"msg":"[2]R flush_numbers[0..12]: [9, 9, 6, 3, 3, 1, 6, 1, 4, 12, 11, 10]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:18.274941781Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1165,"":"downstairs","":"downstairs","session_id":"477622a3-9fac-45be-b783-6a65254e0141"} {"msg":"[2]R generation[0..12]: [6, 6, 5, 2, 2, 1, 5, 1, 3, 7, 6, 6]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:18.27495178Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1165,"":"downstairs","":"downstairs","session_id":"477622a3-9fac-45be-b783-6a65254e0141"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:18.27496275Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1165,"":"downstairs","":"downstairs","session_id":"477622a3-9fac-45be-b783-6a65254e0141"} {"msg":"Max found gen is 8","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:18.27497216Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1165,"":"downstairs","":"downstairs","session_id":"477622a3-9fac-45be-b783-6a65254e0141"} {"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:18.274982069Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1165,"":"downstairs","":"downstairs","session_id":"477622a3-9fac-45be-b783-6a65254e0141"} {"msg":"Next flush: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:18.274993509Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1165,"":"downstairs","":"downstairs","session_id":"477622a3-9fac-45be-b783-6a65254e0141"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:18.275002839Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1165,"":"downstairs","":"downstairs","session_id":"477622a3-9fac-45be-b783-6a65254e0141"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:18.275012439Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1165,"session_id":"477622a3-9fac-45be-b783-6a65254e0141"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:18.275021168Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1165,"client":"0","":"downstairs","session_id":"477622a3-9fac-45be-b783-6a65254e0141"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:18.275030728Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1165,"client":"1","":"downstairs","session_id":"477622a3-9fac-45be-b783-6a65254e0141"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:18.275040268Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1165,"client":"2","":"downstairs","session_id":"477622a3-9fac-45be-b783-6a65254e0141"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:18.275050078Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1165,"session_id":"477622a3-9fac-45be-b783-6a65254e0141"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:18.275059067Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1165,"session_id":"477622a3-9fac-45be-b783-6a65254e0141"} {"msg":"ef93920d-7f2c-45b7-8343-61f2ad18540a is now active with session: 477622a3-9fac-45be-b783-6a65254e0141","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:18.275068907Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1165,"session_id":"477622a3-9fac-45be-b783-6a65254e0141"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:18.275077737Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1165,"session_id":"477622a3-9fac-45be-b783-6a65254e0141"} {"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:18.275086746Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1165} {"msg":"Activated sub_volume ef93920d-7f2c-45b7-8343-61f2ad18540a","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:18.275094636Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1165} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:18.275102606Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1165,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:18.275414997Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1165,"task":"crutest"} 01/30 Write block 480 len 8192 data: 2 2 02/30 Flush 03/30 Write block 315 len 8192 data: 2 3 04/30 Read block 461 len 16384 05/30 Read block 394 len 4096 06/30 Read block 260 len 36864 07/30 Write block 399 len 4096 data: 3 08/30 Write block 308 len 32768 data: 2 2 2 2 2 2 2 3 09/30 Read block 353 len 28672 10/30 Write block 6 len 24576 data: 2 2 2 2 2 2 11/30 Read block 239 len 40960 12/30 Read block 96 len 8192 13/30 Read block 29 len 8192 14/30 Read block 49 len 40960 15/30 Read block 341 len 24576 16/30 Write block 89 len 8192 data: 2 2 17/30 Flush 18/30 Write block 71 len 32768 data: 2 2 2 2 2 3 3 3 19/30 Write block 94 len 12288 data: 2 2 2 20/30 Write block 293 len 24576 data: 3 3 2 2 2 2 21/30 Write block 336 len 8192 data: 3 4 22/30 Read block 465 len 12288 23/30 Write block 295 len 28672 data: 3 3 3 3 2 2 2 24/30 Write block 222 len 28672 data: 2 3 3 3 3 3 3 25/30 Read block 15 len 12288 26/30 Flush 27/30 Write block 173 len 24576 data: 2 2 2 2 2 2 28/30 Write block 282 len 4096 data: 2 29/30 Write block 133 len 16384 data: 3 3 2 2 30/30 Write block 405 len 12288 data: 2 2 2 ---------------------------------------------------------------- Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 2 Done Done Done false 1003 Acked Read 4 Done Done Done false 1004 Acked Read 1 Sent Done Done false 1005 Acked Read 9 Sent Done Done false 1006 Acked Write 1 Sent Done Done false 1007 Acked Write 8 Sent Done Done false 1008 Acked Read 7 Sent Done Done false 1009 Acked Write 6 Sent Done Done false 1010 Acked Read 10 Sent Done Done false 1011 Acked Read 2 Sent Done Done false 1012 Acked Read 2 Sent Done Done false 1013 Acked Read 10 Sent Done Done false 1014 Acked Read 6 Sent Done Done false 1015 Acked Write 2 Sent Done Done false 1016 Acked Flush 0 Sent Done Done false 1017 Acked Write 8 Sent Done Done false 1018 Acked Write 3 Sent Done Done false 1019 Acked Write 6 Sent Done Done false 1020 Acked Write 2 Sent Done Done false 1021 Acked Read 3 Sent Done Done false 1022 Acked Write 7 Sent Done Done false 1023 Acked Write 7 Sent Done Done false 1024 Acked Read 3 Sent Done Done false 1025 Acked Flush 0 Sent Done Done false 1026 Acked Write 6 Sent Sent Sent false 1027 Acked Write 1 Sent Sent Sent false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 26 4 4 34 Done 2 24 24 50 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1001)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:42:18.283191753Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1165} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:18.283359978Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1165,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 6 8 8 9 14 14 F F F <--- 1 020-039 6 6 6 9 9 9 F F F 2 040-059 5 5 5 6 6 6 F F F 3 060-079 2 8 8 3 15 15 F F F <--- 4 080-099 2 8 8 3 15 15 F F F <--- 5 100-119 1 1 1 1 1 1 F F F 6 120-139 5 5 5 6 6 6 F T T <--- 7 140-159 1 1 1 1 1 1 F F F 8 160-179 3 3 3 4 4 4 F T T <--- 9 180-199 7 7 7 12 12 12 F F F 10 200-219 6 6 6 11 11 11 F F F 11 220-239 6 8 8 10 15 15 F F F <--- 12 240-259 5 5 5 6 6 6 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 4 8 8 5 15 15 F T T <--- 15 300-319 6 8 8 9 15 15 T F F <--- 16 320-339 6 8 8 11 15 15 F F F <--- 17 340-359 6 6 6 11 11 11 F F F 18 360-379 6 6 6 9 9 9 F F F 19 380-399 6 8 8 9 14 14 F F F <--- 20 400-419 6 6 6 9 9 9 F T T <--- 21 420-439 7 7 7 12 12 12 F F F 22 440-459 6 6 6 9 9 9 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 8 8 8 13 13 13 F F F 25 500-519 7 7 7 12 12 12 F F F 26 520-539 5 5 5 6 6 6 F F F 27 540-559 5 5 5 6 6 6 F F F 28 560-579 6 6 6 9 9 9 F F F 29 580-599 7 7 7 12 12 12 F F F Max gen: 8, Max flush: 15 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 4 Verifying data now No change after verify Loop: 4 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 14 14 14 F F F 1 020-039 6 6 6 9 9 9 F F F 2 040-059 5 5 5 6 6 6 F F F 3 060-079 8 8 8 15 15 15 F F F 4 080-099 8 8 8 15 15 15 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 9 9 9 16 16 16 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 9 9 9 16 16 16 F F F 9 180-199 7 7 7 12 12 12 F F F 10 200-219 6 6 6 11 11 11 F F F 11 220-239 8 8 8 15 15 15 F F F 12 240-259 5 5 5 6 6 6 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 9 9 9 16 16 16 F F F 15 300-319 8 8 8 15 15 15 F F F 16 320-339 8 8 8 15 15 15 F F F 17 340-359 6 6 6 11 11 11 F F F 18 360-379 6 6 6 9 9 9 F F F 19 380-399 8 8 8 14 14 14 F F F 20 400-419 9 9 9 16 16 16 F F F 21 420-439 7 7 7 12 12 12 F F F 22 440-459 6 6 6 9 9 9 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 8 8 8 13 13 13 F F F 25 500-519 7 7 7 12 12 12 F F F 26 520-539 5 5 5 6 6 6 F F F 27 540-559 5 5 5 6 6 6 F F F 28 560-579 6 6 6 9 9 9 F F F 29 580-599 7 7 7 12 12 12 F F F Max gen: 9, Max flush: 16 Begin loop 5 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:22.526302402Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1173,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:22.534444907Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1173,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:22.534731029Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1173} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:22.534760398Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1173} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:22.534771047Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1173} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:22.534782637Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1173} {"msg":"Crucible b0f54b18-9f38-4470-96c0-70388f0bdada has session id: 752bc13f-5c36-4889-a773-0453b9ad9efd","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:22.535159536Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1173,"session_id":"752bc13f-5c36-4889-a773-0453b9ad9efd"} {"msg":"Upstairs opts: Upstairs UUID: b0f54b18-9f38-4470-96c0-70388f0bdada, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:22.535188775Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1173,"session_id":"752bc13f-5c36-4889-a773-0453b9ad9efd"} {"msg":"Crucible stats registered with UUID: b0f54b18-9f38-4470-96c0-70388f0bdada","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:22.535199835Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1173,"session_id":"752bc13f-5c36-4889-a773-0453b9ad9efd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:42:22.535208995Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1173,"session_id":"752bc13f-5c36-4889-a773-0453b9ad9efd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:22.535339621Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1173,"":"io task","client":"2","":"downstairs","session_id":"752bc13f-5c36-4889-a773-0453b9ad9efd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:22.535357371Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1173,"":"io task","client":"0","":"downstairs","session_id":"752bc13f-5c36-4889-a773-0453b9ad9efd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:22.5353675Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1173,"":"io task","client":"1","":"downstairs","session_id":"752bc13f-5c36-4889-a773-0453b9ad9efd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:22.535513436Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1173,"task":"crutest"} {"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:22.535528436Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1173} {"msg":"b0f54b18-9f38-4470-96c0-70388f0bdada active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:22.535566675Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1173,"session_id":"752bc13f-5c36-4889-a773-0453b9ad9efd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:22.535576154Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1173,"client":"0","":"downstairs","session_id":"752bc13f-5c36-4889-a773-0453b9ad9efd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:22.535585764Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1173,"client":"1","":"downstairs","session_id":"752bc13f-5c36-4889-a773-0453b9ad9efd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:22.535595364Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1173,"client":"2","":"downstairs","session_id":"752bc13f-5c36-4889-a773-0453b9ad9efd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:22.535606743Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1173,"":"io task","client":"2","":"downstairs","session_id":"752bc13f-5c36-4889-a773-0453b9ad9efd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:22.535616793Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1173,"":"io task","client":"0","":"downstairs","session_id":"752bc13f-5c36-4889-a773-0453b9ad9efd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:22.535626723Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1173,"":"io task","client":"1","":"downstairs","session_id":"752bc13f-5c36-4889-a773-0453b9ad9efd"} {"msg":"ds_connection connected from Ok(127.0.0.1:50354)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:22.53572138Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1173,"":"io task","client":"2","":"downstairs","session_id":"752bc13f-5c36-4889-a773-0453b9ad9efd"} {"msg":"ds_connection connected from Ok(127.0.0.1:46382)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:22.53573382Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1173,"":"io task","client":"1","":"downstairs","session_id":"752bc13f-5c36-4889-a773-0453b9ad9efd"} {"msg":"ds_connection connected from Ok(127.0.0.1:61126)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:22.535744019Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1173,"":"io task","client":"0","":"downstairs","session_id":"752bc13f-5c36-4889-a773-0453b9ad9efd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:22.536367401Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1173,"client":"1","":"downstairs","session_id":"752bc13f-5c36-4889-a773-0453b9ad9efd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:22.536476338Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1173,"client":"0","":"downstairs","session_id":"752bc13f-5c36-4889-a773-0453b9ad9efd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:22.536540746Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1173,"client":"2","":"downstairs","session_id":"752bc13f-5c36-4889-a773-0453b9ad9efd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:22.536668923Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1173,"session_id":"752bc13f-5c36-4889-a773-0453b9ad9efd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:22.536742831Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1173,"session_id":"752bc13f-5c36-4889-a773-0453b9ad9efd"} {"msg":"[0]R flush_numbers[0..12]: [14, 9, 6, 15, 15, 1, 16, 1, 16, 12, 11, 15]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:22.536826928Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1173,"":"downstairs","":"downstairs","session_id":"752bc13f-5c36-4889-a773-0453b9ad9efd"} {"msg":"[0]R generation[0..12]: [8, 6, 5, 8, 8, 1, 9, 1, 9, 7, 6, 8]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:22.536837798Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1173,"":"downstairs","":"downstairs","session_id":"752bc13f-5c36-4889-a773-0453b9ad9efd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:22.536848908Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1173,"":"downstairs","":"downstairs","session_id":"752bc13f-5c36-4889-a773-0453b9ad9efd"} {"msg":"[1]R flush_numbers[0..12]: [14, 9, 6, 15, 15, 1, 16, 1, 16, 12, 11, 15]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:22.536861597Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1173,"":"downstairs","":"downstairs","session_id":"752bc13f-5c36-4889-a773-0453b9ad9efd"} {"msg":"[1]R generation[0..12]: [8, 6, 5, 8, 8, 1, 9, 1, 9, 7, 6, 8]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:22.536871847Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1173,"":"downstairs","":"downstairs","session_id":"752bc13f-5c36-4889-a773-0453b9ad9efd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:22.536882837Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1173,"":"downstairs","":"downstairs","session_id":"752bc13f-5c36-4889-a773-0453b9ad9efd"} {"msg":"[2]R flush_numbers[0..12]: [14, 9, 6, 15, 15, 1, 16, 1, 16, 12, 11, 15]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:22.536893066Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1173,"":"downstairs","":"downstairs","session_id":"752bc13f-5c36-4889-a773-0453b9ad9efd"} {"msg":"[2]R generation[0..12]: [8, 6, 5, 8, 8, 1, 9, 1, 9, 7, 6, 8]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:22.536903196Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1173,"":"downstairs","":"downstairs","session_id":"752bc13f-5c36-4889-a773-0453b9ad9efd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:22.536914036Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1173,"":"downstairs","":"downstairs","session_id":"752bc13f-5c36-4889-a773-0453b9ad9efd"} {"msg":"Max found gen is 10","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:22.536923535Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1173,"":"downstairs","":"downstairs","session_id":"752bc13f-5c36-4889-a773-0453b9ad9efd"} {"msg":"Generation requested: 10 >= found:10","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:22.536933095Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1173,"":"downstairs","":"downstairs","session_id":"752bc13f-5c36-4889-a773-0453b9ad9efd"} {"msg":"Next flush: 17","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:22.536942385Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1173,"":"downstairs","":"downstairs","session_id":"752bc13f-5c36-4889-a773-0453b9ad9efd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:22.536951855Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1173,"":"downstairs","":"downstairs","session_id":"752bc13f-5c36-4889-a773-0453b9ad9efd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:22.536961454Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1173,"session_id":"752bc13f-5c36-4889-a773-0453b9ad9efd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:22.536970144Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1173,"client":"0","":"downstairs","session_id":"752bc13f-5c36-4889-a773-0453b9ad9efd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:22.536979644Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1173,"client":"1","":"downstairs","session_id":"752bc13f-5c36-4889-a773-0453b9ad9efd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:22.536989173Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1173,"client":"2","":"downstairs","session_id":"752bc13f-5c36-4889-a773-0453b9ad9efd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:22.537010773Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1173,"session_id":"752bc13f-5c36-4889-a773-0453b9ad9efd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:22.537019983Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1173,"session_id":"752bc13f-5c36-4889-a773-0453b9ad9efd"} {"msg":"b0f54b18-9f38-4470-96c0-70388f0bdada is now active with session: 752bc13f-5c36-4889-a773-0453b9ad9efd","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:22.537031022Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1173,"session_id":"752bc13f-5c36-4889-a773-0453b9ad9efd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:22.537039662Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1173,"session_id":"752bc13f-5c36-4889-a773-0453b9ad9efd"} {"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:22.537048622Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1173} {"msg":"Activated sub_volume b0f54b18-9f38-4470-96c0-70388f0bdada","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:22.537056462Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1173} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:22.537064581Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1173,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:22.537367733Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1173,"task":"crutest"} 01/30 Write block 545 len 4096 data: 2 02/30 Write block 142 len 36864 data: 2 2 2 2 2 2 2 2 2 03/30 Read block 314 len 28672 04/30 Write block 379 len 24576 data: 3 3 3 3 2 2 05/30 Read block 548 len 16384 06/30 Read block 261 len 36864 07/30 Read block 568 len 4096 08/30 Write block 354 len 20480 data: 3 3 3 3 3 09/30 Write block 532 len 8192 data: 3 3 10/30 Read block 554 len 16384 11/30 Read block 83 len 8192 12/30 Write block 250 len 4096 data: 3 13/30 Write block 119 len 24576 data: 2 2 2 2 2 2 14/30 Read block 79 len 32768 15/30 Write block 515 len 4096 data: 2 16/30 Write block 438 len 20480 data: 3 3 3 3 3 17/30 Write block 4 len 28672 data: 2 2 3 3 3 3 3 18/30 Read block 112 len 4096 19/30 Write block 570 len 40960 data: 2 2 2 2 2 2 4 4 4 4 20/30 Read block 156 len 4096 21/30 Read block 200 len 4096 22/30 Write block 544 len 28672 data: 2 3 2 2 3 3 3 23/30 Write block 273 len 20480 data: 2 2 2 2 2 24/30 Read block 85 len 28672 25/30 Read block 477 len 28672 26/30 Write block 235 len 4096 data: 2 27/30 Read block 29 len 20480 28/30 Write block 16 len 8192 data: 2 2 29/30 Read block 29 len 16384 30/30 Read block 240 len 16384 ---------------------------------------------------------------- Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Sent false 1001 Acked Write 9 Done Done Sent false 1002 Acked Read 7 Done Done Sent false 1003 Acked Write 6 Done Done Sent false 1004 Acked Read 4 Done Done Sent false 1005 Acked Read 9 Done Done Sent false 1006 Acked Read 1 Done Done Sent false 1007 Acked Write 5 Done Done Sent false 1008 Acked Write 2 Done Done Sent false 1009 Acked Read 4 Done Done Sent false 1010 Acked Read 2 Done Done Sent false 1011 Acked Write 1 Done Done Sent false 1012 Acked Write 6 Done Done Sent false 1013 Acked Read 8 Done Done Sent false 1014 Acked Write 1 Done Done Sent false 1015 Acked Write 5 Done Done Sent false 1016 Acked Write 7 Done Done Sent false 1017 Acked Read 1 Done Done Sent false 1018 Acked Write 10 Done Done Sent false 1019 Acked Read 1 Done Done Sent false 1020 Acked Read 1 Done Done Sent false 1021 Acked Write 7 Done Done Sent false 1022 Acked Write 5 Done Done Sent false 1023 Acked Read 7 Done Done Sent false 1024 Acked Read 7 Done Done Sent false 1025 Acked Write 1 Done Done Sent false 1026 Acked Read 5 Done Done Sent false 1027 Acked Write 2 Done Done Sent false 1028 Acked Read 4 Done Done Sent false 1029 Acked Read 4 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 30 30 Done 30 30 0 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:42:22.544755259Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1173} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:22.544925685Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1173,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 14 14 14 T T F <--- 1 020-039 6 6 6 9 9 9 F F F 2 040-059 5 5 5 6 6 6 F F F 3 060-079 8 8 8 15 15 15 F F F 4 080-099 8 8 8 15 15 15 F F F 5 100-119 1 1 1 1 1 1 T T F <--- 6 120-139 9 9 9 16 16 16 T T F <--- 7 140-159 1 1 1 1 1 1 T T F <--- 8 160-179 9 9 9 16 16 16 F F F 9 180-199 7 7 7 12 12 12 F F F 10 200-219 6 6 6 11 11 11 F F F 11 220-239 8 8 8 15 15 15 T T F <--- 12 240-259 5 5 5 6 6 6 T T F <--- 13 260-279 1 1 1 1 1 1 T T F <--- 14 280-299 9 9 9 16 16 16 F F F 15 300-319 8 8 8 15 15 15 F F F 16 320-339 8 8 8 15 15 15 F F F 17 340-359 6 6 6 11 11 11 T T F <--- 18 360-379 6 6 6 9 9 9 T T F <--- 19 380-399 8 8 8 14 14 14 T T F <--- 20 400-419 9 9 9 16 16 16 F F F 21 420-439 7 7 7 12 12 12 T T F <--- 22 440-459 6 6 6 9 9 9 T T F <--- 23 460-479 1 1 1 1 1 1 F F F 24 480-499 8 8 8 13 13 13 F F F 25 500-519 7 7 7 12 12 12 T T F <--- 26 520-539 5 5 5 6 6 6 T T F <--- 27 540-559 5 5 5 6 6 6 T T F <--- 28 560-579 6 6 6 9 9 9 T T F <--- 29 580-599 7 7 7 12 12 12 F F F Max gen: 9, Max flush: 16 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 5 Verifying data now No change after verify Loop: 5 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 17 17 17 F F F 1 020-039 6 6 6 9 9 9 F F F 2 040-059 5 5 5 6 6 6 F F F 3 060-079 8 8 8 15 15 15 F F F 4 080-099 8 8 8 15 15 15 F F F 5 100-119 10 10 10 17 17 17 F F F 6 120-139 10 10 10 17 17 17 F F F 7 140-159 10 10 10 17 17 17 F F F 8 160-179 9 9 9 16 16 16 F F F 9 180-199 7 7 7 12 12 12 F F F 10 200-219 6 6 6 11 11 11 F F F 11 220-239 10 10 10 17 17 17 F F F 12 240-259 10 10 10 17 17 17 F F F 13 260-279 10 10 10 17 17 17 F F F 14 280-299 9 9 9 16 16 16 F F F 15 300-319 8 8 8 15 15 15 F F F 16 320-339 8 8 8 15 15 15 F F F 17 340-359 10 10 10 17 17 17 F F F 18 360-379 10 10 10 17 17 17 F F F 19 380-399 10 10 10 17 17 17 F F F 20 400-419 9 9 9 16 16 16 F F F 21 420-439 10 10 10 17 17 17 F F F 22 440-459 10 10 10 17 17 17 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 8 8 8 13 13 13 F F F 25 500-519 10 10 10 17 17 17 F F F 26 520-539 10 10 10 17 17 17 F F F 27 540-559 10 10 10 17 17 17 F F F 28 560-579 10 10 10 17 17 17 F F F 29 580-599 7 7 7 12 12 12 F F F Max gen: 10, Max flush: 17 Begin loop 6 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:26.801645371Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1181,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:26.80965947Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1181,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:26.809976261Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1181} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:26.81001471Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1181} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:26.810023849Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1181} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:26.810035709Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1181} {"msg":"Crucible 2a68f44a-53b0-4383-87f0-fb3fa371f48d has session id: 7ae8fffa-94eb-4b1a-8104-e2e5d4d12039","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:26.810307071Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1181,"session_id":"7ae8fffa-94eb-4b1a-8104-e2e5d4d12039"} {"msg":"Upstairs opts: Upstairs UUID: 2a68f44a-53b0-4383-87f0-fb3fa371f48d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:26.81035368Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1181,"session_id":"7ae8fffa-94eb-4b1a-8104-e2e5d4d12039"} {"msg":"Crucible stats registered with UUID: 2a68f44a-53b0-4383-87f0-fb3fa371f48d","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:26.81036443Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1181,"session_id":"7ae8fffa-94eb-4b1a-8104-e2e5d4d12039"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:42:26.810376229Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1181,"session_id":"7ae8fffa-94eb-4b1a-8104-e2e5d4d12039"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:26.810509345Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1181,"":"io task","client":"1","":"downstairs","session_id":"7ae8fffa-94eb-4b1a-8104-e2e5d4d12039"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:26.810541724Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1181,"":"io task","client":"0","":"downstairs","session_id":"7ae8fffa-94eb-4b1a-8104-e2e5d4d12039"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:26.810552624Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1181,"":"io task","client":"2","":"downstairs","session_id":"7ae8fffa-94eb-4b1a-8104-e2e5d4d12039"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:26.810671931Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1181,"task":"crutest"} {"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:26.81069039Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1181} {"msg":"2a68f44a-53b0-4383-87f0-fb3fa371f48d active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:26.810719469Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1181,"session_id":"7ae8fffa-94eb-4b1a-8104-e2e5d4d12039"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:26.810728779Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1181,"client":"0","":"downstairs","session_id":"7ae8fffa-94eb-4b1a-8104-e2e5d4d12039"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:26.810738419Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1181,"client":"1","":"downstairs","session_id":"7ae8fffa-94eb-4b1a-8104-e2e5d4d12039"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:26.810747879Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1181,"client":"2","":"downstairs","session_id":"7ae8fffa-94eb-4b1a-8104-e2e5d4d12039"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:26.810757288Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1181,"":"io task","client":"2","":"downstairs","session_id":"7ae8fffa-94eb-4b1a-8104-e2e5d4d12039"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:26.810767248Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1181,"":"io task","client":"0","":"downstairs","session_id":"7ae8fffa-94eb-4b1a-8104-e2e5d4d12039"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:26.810777088Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1181,"":"io task","client":"1","":"downstairs","session_id":"7ae8fffa-94eb-4b1a-8104-e2e5d4d12039"} {"msg":"ds_connection connected from Ok(127.0.0.1:58299)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:26.810883155Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1181,"":"io task","client":"2","":"downstairs","session_id":"7ae8fffa-94eb-4b1a-8104-e2e5d4d12039"} {"msg":"ds_connection connected from Ok(127.0.0.1:54703)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:26.810966742Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1181,"":"io task","client":"1","":"downstairs","session_id":"7ae8fffa-94eb-4b1a-8104-e2e5d4d12039"} {"msg":"ds_connection connected from Ok(127.0.0.1:60491)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:26.810986992Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1181,"":"io task","client":"0","":"downstairs","session_id":"7ae8fffa-94eb-4b1a-8104-e2e5d4d12039"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:26.811526636Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1181,"client":"2","":"downstairs","session_id":"7ae8fffa-94eb-4b1a-8104-e2e5d4d12039"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:26.811547875Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1181,"client":"1","":"downstairs","session_id":"7ae8fffa-94eb-4b1a-8104-e2e5d4d12039"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:26.811641573Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1181,"client":"0","":"downstairs","session_id":"7ae8fffa-94eb-4b1a-8104-e2e5d4d12039"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:26.811850107Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1181,"session_id":"7ae8fffa-94eb-4b1a-8104-e2e5d4d12039"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:26.811886816Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1181,"session_id":"7ae8fffa-94eb-4b1a-8104-e2e5d4d12039"} {"msg":"[0]R flush_numbers[0..12]: [17, 9, 6, 15, 15, 17, 17, 17, 16, 12, 11, 17]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:26.811941174Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1181,"":"downstairs","":"downstairs","session_id":"7ae8fffa-94eb-4b1a-8104-e2e5d4d12039"} {"msg":"[0]R generation[0..12]: [10, 6, 5, 8, 8, 10, 10, 10, 9, 7, 6, 10]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:26.811953324Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1181,"":"downstairs","":"downstairs","session_id":"7ae8fffa-94eb-4b1a-8104-e2e5d4d12039"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:26.811964483Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1181,"":"downstairs","":"downstairs","session_id":"7ae8fffa-94eb-4b1a-8104-e2e5d4d12039"} {"msg":"[1]R flush_numbers[0..12]: [17, 9, 6, 15, 15, 17, 17, 17, 16, 12, 11, 17]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:26.811974993Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1181,"":"downstairs","":"downstairs","session_id":"7ae8fffa-94eb-4b1a-8104-e2e5d4d12039"} {"msg":"[1]R generation[0..12]: [10, 6, 5, 8, 8, 10, 10, 10, 9, 7, 6, 10]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:26.811985283Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1181,"":"downstairs","":"downstairs","session_id":"7ae8fffa-94eb-4b1a-8104-e2e5d4d12039"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:26.811996213Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1181,"":"downstairs","":"downstairs","session_id":"7ae8fffa-94eb-4b1a-8104-e2e5d4d12039"} {"msg":"[2]R flush_numbers[0..12]: [17, 9, 6, 15, 15, 17, 17, 17, 16, 12, 11, 17]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:26.812006582Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1181,"":"downstairs","":"downstairs","session_id":"7ae8fffa-94eb-4b1a-8104-e2e5d4d12039"} {"msg":"[2]R generation[0..12]: [10, 6, 5, 8, 8, 10, 10, 10, 9, 7, 6, 10]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:26.812016812Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1181,"":"downstairs","":"downstairs","session_id":"7ae8fffa-94eb-4b1a-8104-e2e5d4d12039"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:26.812027632Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1181,"":"downstairs","":"downstairs","session_id":"7ae8fffa-94eb-4b1a-8104-e2e5d4d12039"} {"msg":"Max found gen is 11","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:26.812038861Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1181,"":"downstairs","":"downstairs","session_id":"7ae8fffa-94eb-4b1a-8104-e2e5d4d12039"} {"msg":"Generation requested: 12 >= found:11","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:26.812048561Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1181,"":"downstairs","":"downstairs","session_id":"7ae8fffa-94eb-4b1a-8104-e2e5d4d12039"} {"msg":"Next flush: 18","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:26.812057811Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1181,"":"downstairs","":"downstairs","session_id":"7ae8fffa-94eb-4b1a-8104-e2e5d4d12039"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:26.81206723Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1181,"":"downstairs","":"downstairs","session_id":"7ae8fffa-94eb-4b1a-8104-e2e5d4d12039"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:26.81207708Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1181,"session_id":"7ae8fffa-94eb-4b1a-8104-e2e5d4d12039"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:26.81208567Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1181,"client":"0","":"downstairs","session_id":"7ae8fffa-94eb-4b1a-8104-e2e5d4d12039"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:26.81209537Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1181,"client":"1","":"downstairs","session_id":"7ae8fffa-94eb-4b1a-8104-e2e5d4d12039"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:26.812104869Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1181,"client":"2","":"downstairs","session_id":"7ae8fffa-94eb-4b1a-8104-e2e5d4d12039"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:26.812114579Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1181,"session_id":"7ae8fffa-94eb-4b1a-8104-e2e5d4d12039"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:26.812123579Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1181,"session_id":"7ae8fffa-94eb-4b1a-8104-e2e5d4d12039"} {"msg":"2a68f44a-53b0-4383-87f0-fb3fa371f48d is now active with session: 7ae8fffa-94eb-4b1a-8104-e2e5d4d12039","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:26.812133339Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1181,"session_id":"7ae8fffa-94eb-4b1a-8104-e2e5d4d12039"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:26.812142028Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1181,"session_id":"7ae8fffa-94eb-4b1a-8104-e2e5d4d12039"} {"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:26.812151008Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1181} {"msg":"Activated sub_volume 2a68f44a-53b0-4383-87f0-fb3fa371f48d","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:26.812159268Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1181} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:26.812167368Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1181,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:26.812505258Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1181,"task":"crutest"} 01/30 Write block 534 len 36864 data: 3 3 2 2 2 2 2 2 2 02/30 Read block 565 len 32768 03/30 Read block 147 len 28672 04/30 Read block 471 len 28672 05/30 Write block 158 len 8192 data: 2 2 06/30 Write block 505 len 28672 data: 2 2 2 2 2 2 3 07/30 Write block 54 len 36864 data: 2 2 2 2 3 2 2 2 2 08/30 Read block 45 len 24576 09/30 Read block 454 len 28672 10/30 Read block 391 len 12288 11/30 Flush 12/30 Write block 0 len 36864 data: 2 2 2 2 3 3 4 4 4 13/30 Write block 556 len 16384 data: 2 3 3 3 14/30 Read block 88 len 8192 15/30 Flush 16/30 Write block 475 len 4096 data: 2 17/30 Write block 165 len 36864 data: 3 2 2 2 3 3 3 2 3 18/30 Read block 598 len 4096 19/30 Write block 5 len 16384 data: 4 5 5 5 20/30 Write block 206 len 32768 data: 2 2 2 2 3 3 4 3 21/30 Write block 363 len 28672 data: 3 4 3 3 3 2 2 22/30 Flush 23/30 Write block 510 len 8192 data: 3 4 24/30 Read block 428 len 20480 25/30 Flush 26/30 Write block 216 len 16384 data: 4 4 2 2 27/30 Read block 440 len 36864 28/30 Write block 520 len 4096 data: 2 29/30 Read block 148 len 32768 30/30 Write block 516 len 4096 data: 2 ---------------------------------------------------------------- Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Sent Done false 1001 Acked Read 8 Done Sent Done false 1002 Acked Read 7 Done Sent Done false 1003 Acked Read 7 Done Sent Done false 1004 Acked Write 2 Done Sent Done false 1005 Acked Write 7 Done Sent Done false 1006 Acked Write 9 Done Sent Done false 1007 Acked Read 6 Done Sent Done false 1008 Acked Read 7 Done Sent Done false 1009 Acked Read 3 Done Sent Done false 1010 Acked Flush 0 Done Sent Done false 1011 Acked Write 9 Done Sent Done false 1012 Acked Write 4 Done Sent Done false 1013 Acked Read 2 Done Sent Done false 1014 Acked Flush 0 Done Sent Done false 1015 Acked Write 1 Done Sent Done false 1016 Acked Write 9 Done Sent Done false 1017 Acked Read 1 Done Sent Done false 1018 Acked Write 4 Done Sent Done false 1019 Acked Write 8 Done Sent Done false 1020 Acked Write 7 Done Sent Done false 1021 Acked Flush 0 Done Sent Done false 1022 Acked Write 2 Done Sent Done false 1023 Acked Read 5 Done Sent Done false 1024 Acked Flush 0 Done Sent Done false 1025 Acked Write 4 Done Sent Done false 1026 Acked Read 9 Done Sent Done false 1027 Acked Write 1 Done Sent Done false 1028 Acked Read 8 Done Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 30 2 33 Done 29 0 28 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) None Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:42:26.821543197Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1181} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:26.8217795Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1181,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 12 10 12 20 17 20 F F F <--- 1 020-039 6 6 6 9 9 9 F F F 2 040-059 12 5 12 18 6 18 F F F <--- 3 060-079 12 8 12 18 15 18 F F F <--- 4 080-099 8 8 8 15 15 15 F F F 5 100-119 10 10 10 17 17 17 F F F 6 120-139 10 10 10 17 17 17 F F F 7 140-159 12 10 12 18 17 18 F F F <--- 8 160-179 12 9 12 20 16 20 F F F <--- 9 180-199 7 7 7 12 12 12 F F F 10 200-219 12 6 12 20 11 20 T F T <--- 11 220-239 10 10 10 17 17 17 F F F 12 240-259 10 10 10 17 17 17 F F F 13 260-279 10 10 10 17 17 17 F F F 14 280-299 9 9 9 16 16 16 F F F 15 300-319 8 8 8 15 15 15 F F F 16 320-339 8 8 8 15 15 15 F F F 17 340-359 10 10 10 17 17 17 F F F 18 360-379 12 10 12 20 17 20 F F F <--- 19 380-399 10 10 10 17 17 17 F F F 20 400-419 9 9 9 16 16 16 F F F 21 420-439 10 10 10 17 17 17 F F F 22 440-459 10 10 10 17 17 17 F F F 23 460-479 12 1 12 20 1 20 F F F <--- 24 480-499 8 8 8 13 13 13 F F F 25 500-519 12 10 12 21 17 21 T F T <--- 26 520-539 12 10 12 18 17 18 T F T <--- 27 540-559 12 10 12 19 17 19 F F F <--- 28 560-579 10 10 10 17 17 17 F F F 29 580-599 7 7 7 12 12 12 F F F Max gen: 12, Max flush: 21 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 6 Verifying data now No change after verify Loop: 6 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 12 12 12 20 20 20 F F F 1 020-039 6 6 6 9 9 9 F F F 2 040-059 12 12 12 18 18 18 F F F 3 060-079 12 12 12 18 18 18 F F F 4 080-099 8 8 8 15 15 15 F F F 5 100-119 10 10 10 17 17 17 F F F 6 120-139 10 10 10 17 17 17 F F F 7 140-159 12 12 12 18 18 18 F F F 8 160-179 12 12 12 20 20 20 F F F 9 180-199 7 7 7 12 12 12 F F F 10 200-219 13 13 13 22 22 22 F F F 11 220-239 10 10 10 17 17 17 F F F 12 240-259 10 10 10 17 17 17 F F F 13 260-279 10 10 10 17 17 17 F F F 14 280-299 9 9 9 16 16 16 F F F 15 300-319 8 8 8 15 15 15 F F F 16 320-339 8 8 8 15 15 15 F F F 17 340-359 10 10 10 17 17 17 F F F 18 360-379 12 12 12 20 20 20 F F F 19 380-399 10 10 10 17 17 17 F F F 20 400-419 9 9 9 16 16 16 F F F 21 420-439 10 10 10 17 17 17 F F F 22 440-459 10 10 10 17 17 17 F F F 23 460-479 12 12 12 20 20 20 F F F 24 480-499 8 8 8 13 13 13 F F F 25 500-519 13 13 13 22 22 22 F F F 26 520-539 13 13 13 22 22 22 F F F 27 540-559 12 12 12 19 19 19 F F F 28 560-579 10 10 10 17 17 17 F F F 29 580-599 7 7 7 12 12 12 F F F Max gen: 13, Max flush: 22 Begin loop 7 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:31.059565113Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1189,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:31.067536393Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1189,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:31.067850174Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1189} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:31.067876003Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1189} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:31.067884553Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1189} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:31.067895982Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1189} {"msg":"Crucible 45b60826-8e77-4a57-94f1-7aa4e783baf8 has session id: 1482367c-41b9-4a82-972d-8966f71766e2","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:31.068297641Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1189,"session_id":"1482367c-41b9-4a82-972d-8966f71766e2"} {"msg":"Upstairs opts: Upstairs UUID: 45b60826-8e77-4a57-94f1-7aa4e783baf8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:31.06832693Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1189,"session_id":"1482367c-41b9-4a82-972d-8966f71766e2"} {"msg":"Crucible stats registered with UUID: 45b60826-8e77-4a57-94f1-7aa4e783baf8","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:31.068337909Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1189,"session_id":"1482367c-41b9-4a82-972d-8966f71766e2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:42:31.068347069Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1189,"session_id":"1482367c-41b9-4a82-972d-8966f71766e2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:31.068481395Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1189,"":"io task","client":"2","":"downstairs","session_id":"1482367c-41b9-4a82-972d-8966f71766e2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:31.068498925Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1189,"":"io task","client":"0","":"downstairs","session_id":"1482367c-41b9-4a82-972d-8966f71766e2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:31.068508935Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1189,"":"io task","client":"1","":"downstairs","session_id":"1482367c-41b9-4a82-972d-8966f71766e2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:31.068631011Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1189,"task":"crutest"} {"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:31.068646491Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1189} {"msg":"45b60826-8e77-4a57-94f1-7aa4e783baf8 active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:31.068683169Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1189,"session_id":"1482367c-41b9-4a82-972d-8966f71766e2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:31.068694909Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1189,"client":"0","":"downstairs","session_id":"1482367c-41b9-4a82-972d-8966f71766e2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:31.068704709Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1189,"client":"1","":"downstairs","session_id":"1482367c-41b9-4a82-972d-8966f71766e2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:31.068714379Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1189,"client":"2","":"downstairs","session_id":"1482367c-41b9-4a82-972d-8966f71766e2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:31.068723668Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1189,"":"io task","client":"2","":"downstairs","session_id":"1482367c-41b9-4a82-972d-8966f71766e2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:31.068733468Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1189,"":"io task","client":"0","":"downstairs","session_id":"1482367c-41b9-4a82-972d-8966f71766e2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:31.068743408Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1189,"":"io task","client":"1","":"downstairs","session_id":"1482367c-41b9-4a82-972d-8966f71766e2"} {"msg":"ds_connection connected from Ok(127.0.0.1:52314)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:31.068789826Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1189,"":"io task","client":"1","":"downstairs","session_id":"1482367c-41b9-4a82-972d-8966f71766e2"} {"msg":"ds_connection connected from Ok(127.0.0.1:45619)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:31.068800596Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1189,"":"io task","client":"2","":"downstairs","session_id":"1482367c-41b9-4a82-972d-8966f71766e2"} {"msg":"ds_connection connected from Ok(127.0.0.1:36248)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:31.068810836Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1189,"":"io task","client":"0","":"downstairs","session_id":"1482367c-41b9-4a82-972d-8966f71766e2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:31.069403199Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1189,"client":"1","":"downstairs","session_id":"1482367c-41b9-4a82-972d-8966f71766e2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:31.069429168Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1189,"client":"0","":"downstairs","session_id":"1482367c-41b9-4a82-972d-8966f71766e2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:31.069519675Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1189,"client":"2","":"downstairs","session_id":"1482367c-41b9-4a82-972d-8966f71766e2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:31.069674641Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1189,"session_id":"1482367c-41b9-4a82-972d-8966f71766e2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:31.069781358Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1189,"session_id":"1482367c-41b9-4a82-972d-8966f71766e2"} {"msg":"[0]R flush_numbers[0..12]: [20, 9, 18, 18, 15, 17, 17, 18, 20, 12, 22, 17]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:31.069849056Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1189,"":"downstairs","":"downstairs","session_id":"1482367c-41b9-4a82-972d-8966f71766e2"} {"msg":"[0]R generation[0..12]: [12, 6, 12, 12, 8, 10, 10, 12, 12, 7, 13, 10]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:31.069862375Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1189,"":"downstairs","":"downstairs","session_id":"1482367c-41b9-4a82-972d-8966f71766e2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:31.069873335Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1189,"":"downstairs","":"downstairs","session_id":"1482367c-41b9-4a82-972d-8966f71766e2"} {"msg":"[1]R flush_numbers[0..12]: [20, 9, 18, 18, 15, 17, 17, 18, 20, 12, 22, 17]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:31.069883745Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1189,"":"downstairs","":"downstairs","session_id":"1482367c-41b9-4a82-972d-8966f71766e2"} {"msg":"[1]R generation[0..12]: [12, 6, 12, 12, 8, 10, 10, 12, 12, 7, 13, 10]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:31.069894065Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1189,"":"downstairs","":"downstairs","session_id":"1482367c-41b9-4a82-972d-8966f71766e2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:31.069904804Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1189,"":"downstairs","":"downstairs","session_id":"1482367c-41b9-4a82-972d-8966f71766e2"} {"msg":"[2]R flush_numbers[0..12]: [20, 9, 18, 18, 15, 17, 17, 18, 20, 12, 22, 17]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:31.069915184Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1189,"":"downstairs","":"downstairs","session_id":"1482367c-41b9-4a82-972d-8966f71766e2"} {"msg":"[2]R generation[0..12]: [12, 6, 12, 12, 8, 10, 10, 12, 12, 7, 13, 10]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:31.069925464Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1189,"":"downstairs","":"downstairs","session_id":"1482367c-41b9-4a82-972d-8966f71766e2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:31.069936183Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1189,"":"downstairs","":"downstairs","session_id":"1482367c-41b9-4a82-972d-8966f71766e2"} {"msg":"Max found gen is 14","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:31.069945513Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1189,"":"downstairs","":"downstairs","session_id":"1482367c-41b9-4a82-972d-8966f71766e2"} {"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:31.069955113Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1189,"":"downstairs","":"downstairs","session_id":"1482367c-41b9-4a82-972d-8966f71766e2"} {"msg":"Next flush: 23","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:31.069964423Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1189,"":"downstairs","":"downstairs","session_id":"1482367c-41b9-4a82-972d-8966f71766e2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:31.069973772Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1189,"":"downstairs","":"downstairs","session_id":"1482367c-41b9-4a82-972d-8966f71766e2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:31.069983342Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1189,"session_id":"1482367c-41b9-4a82-972d-8966f71766e2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:31.069991922Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1189,"client":"0","":"downstairs","session_id":"1482367c-41b9-4a82-972d-8966f71766e2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:31.070001451Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1189,"client":"1","":"downstairs","session_id":"1482367c-41b9-4a82-972d-8966f71766e2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:31.070012291Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1189,"client":"2","":"downstairs","session_id":"1482367c-41b9-4a82-972d-8966f71766e2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:31.070021911Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1189,"session_id":"1482367c-41b9-4a82-972d-8966f71766e2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:31.070030901Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1189,"session_id":"1482367c-41b9-4a82-972d-8966f71766e2"} {"msg":"45b60826-8e77-4a57-94f1-7aa4e783baf8 is now active with session: 1482367c-41b9-4a82-972d-8966f71766e2","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:31.07004069Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1189,"session_id":"1482367c-41b9-4a82-972d-8966f71766e2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:31.07004973Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1189,"session_id":"1482367c-41b9-4a82-972d-8966f71766e2"} {"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:31.07005909Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1189} {"msg":"Activated sub_volume 45b60826-8e77-4a57-94f1-7aa4e783baf8","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:31.07006717Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1189} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:31.070075149Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1189,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:31.070336432Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1189,"task":"crutest"} 01/30 Write block 31 len 40960 data: 2 2 2 2 2 2 2 2 2 2 02/30 Write block 119 len 40960 data: 3 3 3 3 3 3 2 3 3 3 03/30 Flush 04/30 Write block 163 len 4096 data: 3 05/30 Read block 3 len 32768 06/30 Read block 381 len 16384 07/30 Write block 539 len 8192 data: 3 3 08/30 Read block 572 len 32768 09/30 Write block 294 len 20480 data: 4 4 4 4 4 10/30 Write block 520 len 40960 data: 3 3 3 3 3 3 2 2 2 2 11/30 Write block 405 len 36864 data: 3 3 3 2 2 2 3 3 2 12/30 Flush 13/30 Write block 375 len 24576 data: 3 3 3 3 4 4 14/30 Read block 565 len 4096 15/30 Read block 346 len 40960 16/30 Read block 397 len 24576 17/30 Read block 188 len 16384 18/30 Read block 231 len 32768 19/30 Read block 298 len 36864 20/30 Read block 169 len 4096 21/30 Write block 491 len 12288 data: 2 2 2 22/30 Flush 23/30 Write block 6 len 40960 data: 6 6 6 4 4 3 2 2 2 2 24/30 Write block 561 len 8192 data: 2 2 25/30 Read block 213 len 36864 26/30 Write block 223 len 36864 data: 4 4 4 4 4 4 3 3 2 27/30 Write block 487 len 8192 data: 3 2 28/30 Write block 201 len 16384 data: 2 2 2 2 29/30 Write block 112 len 36864 data: 2 2 2 2 2 2 2 4 4 30/30 Write block 413 len 8192 data: 3 2 ---------------------------------------------------------------- Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Sent false 1001 Acked Write 10 Done Done Sent false 1002 Acked Flush 0 Done Done Sent false 1003 Acked Write 1 Done Done Sent false 1004 Acked Read 8 Done Done Sent false 1005 Acked Read 4 Done Done Sent false 1006 Acked Write 2 Done Done Sent false 1007 Acked Read 8 Done Done Sent false 1008 Acked Write 5 Done Done Sent false 1009 Acked Write 10 Done Done Sent false 1010 Acked Write 9 Done Done Sent false 1011 Acked Flush 0 Done Done Sent false 1012 Acked Write 6 Done Done Sent false 1013 Acked Read 1 Done Done Sent false 1014 Acked Read 10 Done Done Sent false 1015 Acked Read 6 Done Done Sent false 1016 Acked Read 4 Done Done Sent false 1017 Acked Read 8 Done Done Sent false 1018 Acked Read 9 Done Done Sent false 1019 Acked Read 1 Done Done Sent false 1020 Acked Write 3 Done Done Sent false 1021 Acked Flush 0 Done Done Sent false 1022 Acked Write 10 Done Done Sent false 1023 Acked Write 2 Done Done Sent false 1024 Acked Read 9 Done Done Sent false 1025 Acked Write 9 Done Done Sent false 1026 Acked Write 2 Done Done Sent false 1027 Acked Write 4 Done Sent Sent false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 3 30 35 Done 28 27 0 55 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:42:31.078896185Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1189} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:31.07906739Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1189,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 12 12 12 20 20 20 T T F <--- 1 020-039 14 14 6 23 23 9 F F F <--- 2 040-059 14 14 12 23 23 18 F F F <--- 3 060-079 12 12 12 18 18 18 F F F 4 080-099 8 8 8 15 15 15 F F F 5 100-119 14 14 10 23 23 17 T T F <--- 6 120-139 14 14 10 23 23 17 T T F <--- 7 140-159 12 12 12 18 18 18 F F F 8 160-179 14 14 12 24 24 20 F F F <--- 9 180-199 7 7 7 12 12 12 F F F 10 200-219 13 13 13 22 22 22 T T F <--- 11 220-239 10 10 10 17 17 17 T T F <--- 12 240-259 10 10 10 17 17 17 F F F 13 260-279 10 10 10 17 17 17 F F F 14 280-299 14 14 9 24 24 16 F F F <--- 15 300-319 8 8 8 15 15 15 F F F 16 320-339 8 8 8 15 15 15 F F F 17 340-359 10 10 10 17 17 17 F F F 18 360-379 14 14 12 25 25 20 F F F <--- 19 380-399 14 14 10 25 25 17 F F F <--- 20 400-419 14 14 9 24 24 16 T T F <--- 21 420-439 10 10 10 17 17 17 F F F 22 440-459 10 10 10 17 17 17 F F F 23 460-479 12 12 12 20 20 20 F F F 24 480-499 14 14 8 25 25 13 T T F <--- 25 500-519 13 13 13 22 22 22 F F F 26 520-539 14 14 13 24 24 22 F F F <--- 27 540-559 14 14 12 24 24 19 F F F <--- 28 560-579 10 10 10 17 17 17 T T F <--- 29 580-599 7 7 7 12 12 12 F F F Max gen: 14, Max flush: 25 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 7 Verifying data now No change after verify Loop: 7 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 15 15 15 26 26 26 F F F 1 020-039 14 14 14 23 23 23 F F F 2 040-059 14 14 14 23 23 23 F F F 3 060-079 12 12 12 18 18 18 F F F 4 080-099 8 8 8 15 15 15 F F F 5 100-119 15 15 15 26 26 26 F F F 6 120-139 15 15 15 26 26 26 F F F 7 140-159 12 12 12 18 18 18 F F F 8 160-179 14 14 14 24 24 24 F F F 9 180-199 7 7 7 12 12 12 F F F 10 200-219 15 15 15 26 26 26 F F F 11 220-239 15 15 15 26 26 26 F F F 12 240-259 10 10 10 17 17 17 F F F 13 260-279 10 10 10 17 17 17 F F F 14 280-299 14 14 14 24 24 24 F F F 15 300-319 8 8 8 15 15 15 F F F 16 320-339 8 8 8 15 15 15 F F F 17 340-359 10 10 10 17 17 17 F F F 18 360-379 14 14 14 25 25 25 F F F 19 380-399 14 14 14 25 25 25 F F F 20 400-419 15 15 15 26 26 26 F F F 21 420-439 10 10 10 17 17 17 F F F 22 440-459 10 10 10 17 17 17 F F F 23 460-479 12 12 12 20 20 20 F F F 24 480-499 15 15 15 26 26 26 F F F 25 500-519 13 13 13 22 22 22 F F F 26 520-539 14 14 14 24 24 24 F F F 27 540-559 14 14 14 24 24 24 F F F 28 560-579 15 15 15 26 26 26 F F F 29 580-599 7 7 7 12 12 12 F F F Max gen: 15, Max flush: 26 Begin loop 8 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:35.32907863Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1197,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:35.337090949Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1197,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:35.337362691Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1197} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:35.33739063Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1197} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:35.33739956Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1197} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:35.33741106Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1197} {"msg":"Crucible 27273d0f-c810-46c9-a4e7-355d5c3d4c13 has session id: 8732f0e3-cb8a-4c54-9665-4f1420d4516f","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:35.337711601Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1197,"session_id":"8732f0e3-cb8a-4c54-9665-4f1420d4516f"} {"msg":"Upstairs opts: Upstairs UUID: 27273d0f-c810-46c9-a4e7-355d5c3d4c13, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:35.33773947Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1197,"session_id":"8732f0e3-cb8a-4c54-9665-4f1420d4516f"} {"msg":"Crucible stats registered with UUID: 27273d0f-c810-46c9-a4e7-355d5c3d4c13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:35.33774955Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1197,"session_id":"8732f0e3-cb8a-4c54-9665-4f1420d4516f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:42:35.3377586Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1197,"session_id":"8732f0e3-cb8a-4c54-9665-4f1420d4516f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:35.337902325Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1197,"":"io task","client":"1","":"downstairs","session_id":"8732f0e3-cb8a-4c54-9665-4f1420d4516f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:35.337935165Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1197,"":"io task","client":"0","":"downstairs","session_id":"8732f0e3-cb8a-4c54-9665-4f1420d4516f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:35.337945914Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1197,"":"io task","client":"2","":"downstairs","session_id":"8732f0e3-cb8a-4c54-9665-4f1420d4516f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:35.33809534Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1197,"task":"crutest"} {"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:35.33810855Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1197} {"msg":"27273d0f-c810-46c9-a4e7-355d5c3d4c13 active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:35.338143748Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1197,"session_id":"8732f0e3-cb8a-4c54-9665-4f1420d4516f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:35.338153168Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1197,"client":"0","":"downstairs","session_id":"8732f0e3-cb8a-4c54-9665-4f1420d4516f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:35.338163208Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1197,"client":"1","":"downstairs","session_id":"8732f0e3-cb8a-4c54-9665-4f1420d4516f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:35.338172888Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1197,"client":"2","":"downstairs","session_id":"8732f0e3-cb8a-4c54-9665-4f1420d4516f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:35.338182447Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1197,"":"io task","client":"2","":"downstairs","session_id":"8732f0e3-cb8a-4c54-9665-4f1420d4516f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:35.338192357Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1197,"":"io task","client":"0","":"downstairs","session_id":"8732f0e3-cb8a-4c54-9665-4f1420d4516f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:35.338202197Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1197,"":"io task","client":"1","":"downstairs","session_id":"8732f0e3-cb8a-4c54-9665-4f1420d4516f"} {"msg":"ds_connection connected from Ok(127.0.0.1:46534)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:35.338274925Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1197,"":"io task","client":"1","":"downstairs","session_id":"8732f0e3-cb8a-4c54-9665-4f1420d4516f"} {"msg":"ds_connection connected from Ok(127.0.0.1:51159)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:35.338405411Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1197,"":"io task","client":"0","":"downstairs","session_id":"8732f0e3-cb8a-4c54-9665-4f1420d4516f"} {"msg":"ds_connection connected from Ok(127.0.0.1:62671)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:35.33843119Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1197,"":"io task","client":"2","":"downstairs","session_id":"8732f0e3-cb8a-4c54-9665-4f1420d4516f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:35.338946165Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1197,"client":"1","":"downstairs","session_id":"8732f0e3-cb8a-4c54-9665-4f1420d4516f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:35.338992994Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1197,"client":"0","":"downstairs","session_id":"8732f0e3-cb8a-4c54-9665-4f1420d4516f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:35.339030303Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1197,"client":"2","":"downstairs","session_id":"8732f0e3-cb8a-4c54-9665-4f1420d4516f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:35.339273936Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1197,"session_id":"8732f0e3-cb8a-4c54-9665-4f1420d4516f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:35.339308445Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1197,"session_id":"8732f0e3-cb8a-4c54-9665-4f1420d4516f"} {"msg":"[0]R flush_numbers[0..12]: [26, 23, 23, 18, 15, 26, 26, 18, 24, 12, 26, 26]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:35.339369363Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1197,"":"downstairs","":"downstairs","session_id":"8732f0e3-cb8a-4c54-9665-4f1420d4516f"} {"msg":"[0]R generation[0..12]: [15, 14, 14, 12, 8, 15, 15, 12, 14, 7, 15, 15]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:35.339380313Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1197,"":"downstairs","":"downstairs","session_id":"8732f0e3-cb8a-4c54-9665-4f1420d4516f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:35.339391253Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1197,"":"downstairs","":"downstairs","session_id":"8732f0e3-cb8a-4c54-9665-4f1420d4516f"} {"msg":"[1]R flush_numbers[0..12]: [26, 23, 23, 18, 15, 26, 26, 18, 24, 12, 26, 26]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:35.339401732Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1197,"":"downstairs","":"downstairs","session_id":"8732f0e3-cb8a-4c54-9665-4f1420d4516f"} {"msg":"[1]R generation[0..12]: [15, 14, 14, 12, 8, 15, 15, 12, 14, 7, 15, 15]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:35.339412042Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1197,"":"downstairs","":"downstairs","session_id":"8732f0e3-cb8a-4c54-9665-4f1420d4516f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:35.339422902Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1197,"":"downstairs","":"downstairs","session_id":"8732f0e3-cb8a-4c54-9665-4f1420d4516f"} {"msg":"[2]R flush_numbers[0..12]: [26, 23, 23, 18, 15, 26, 26, 18, 24, 12, 26, 26]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:35.339435291Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1197,"":"downstairs","":"downstairs","session_id":"8732f0e3-cb8a-4c54-9665-4f1420d4516f"} {"msg":"[2]R generation[0..12]: [15, 14, 14, 12, 8, 15, 15, 12, 14, 7, 15, 15]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:35.339445641Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1197,"":"downstairs","":"downstairs","session_id":"8732f0e3-cb8a-4c54-9665-4f1420d4516f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:35.339456531Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1197,"":"downstairs","":"downstairs","session_id":"8732f0e3-cb8a-4c54-9665-4f1420d4516f"} {"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:35.33946588Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1197,"":"downstairs","":"downstairs","session_id":"8732f0e3-cb8a-4c54-9665-4f1420d4516f"} {"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:35.33947566Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1197,"":"downstairs","":"downstairs","session_id":"8732f0e3-cb8a-4c54-9665-4f1420d4516f"} {"msg":"Next flush: 27","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:35.33948481Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1197,"":"downstairs","":"downstairs","session_id":"8732f0e3-cb8a-4c54-9665-4f1420d4516f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:35.33949417Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1197,"":"downstairs","":"downstairs","session_id":"8732f0e3-cb8a-4c54-9665-4f1420d4516f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:35.339503819Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1197,"session_id":"8732f0e3-cb8a-4c54-9665-4f1420d4516f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:35.339512469Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1197,"client":"0","":"downstairs","session_id":"8732f0e3-cb8a-4c54-9665-4f1420d4516f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:35.339522139Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1197,"client":"1","":"downstairs","session_id":"8732f0e3-cb8a-4c54-9665-4f1420d4516f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:35.339531678Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1197,"client":"2","":"downstairs","session_id":"8732f0e3-cb8a-4c54-9665-4f1420d4516f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:35.339541518Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1197,"session_id":"8732f0e3-cb8a-4c54-9665-4f1420d4516f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:35.339550618Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1197,"session_id":"8732f0e3-cb8a-4c54-9665-4f1420d4516f"} {"msg":"27273d0f-c810-46c9-a4e7-355d5c3d4c13 is now active with session: 8732f0e3-cb8a-4c54-9665-4f1420d4516f","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:35.339560548Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1197,"session_id":"8732f0e3-cb8a-4c54-9665-4f1420d4516f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:35.339569217Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1197,"session_id":"8732f0e3-cb8a-4c54-9665-4f1420d4516f"} {"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:35.339578197Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1197} {"msg":"Activated sub_volume 27273d0f-c810-46c9-a4e7-355d5c3d4c13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:35.339587527Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1197} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:35.339595677Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1197,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:35.339895798Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1197,"task":"crutest"} 01/30 Write block 146 len 12288 data: 3 3 3 02/30 Read block 39 len 40960 03/30 Read block 389 len 20480 04/30 Flush 05/30 Write block 521 len 16384 data: 4 4 4 4 06/30 Read block 85 len 40960 07/30 Read block 86 len 8192 08/30 Flush 09/30 Write block 22 len 4096 data: 2 10/30 Write block 543 len 4096 data: 2 11/30 Flush 12/30 Write block 507 len 24576 data: 3 3 3 4 5 3 13/30 Read block 160 len 28672 14/30 Read block 97 len 28672 15/30 Flush 16/30 Flush 17/30 Write block 113 len 28672 data: 3 3 3 3 3 3 5 18/30 Read block 212 len 12288 19/30 Write block 174 len 16384 data: 3 3 3 3 20/30 Read block 589 len 12288 21/30 Write block 201 len 40960 data: 3 3 3 3 2 3 3 3 3 4 22/30 Flush 23/30 Write block 28 len 24576 data: 2 2 2 3 3 3 24/30 Read block 324 len 20480 25/30 Read block 234 len 8192 26/30 Write block 550 len 16384 data: 4 2 2 2 27/30 Write block 135 len 28672 data: 3 3 2 2 2 2 2 28/30 Read block 150 len 28672 29/30 Read block 497 len 12288 30/30 Read block 421 len 8192 ---------------------------------------------------------------- Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Write 4 Done Done Done false 1005 Acked Read 10 Done Done Done false 1006 Acked Read 2 Done Done Sent false 1007 Acked Flush 0 Done Done Sent false 1008 Acked Write 1 Done Done Sent false 1009 Acked Write 1 Done Done Sent false 1010 Acked Flush 0 Done Done Sent false 1011 Acked Write 6 Done Done Sent false 1012 Acked Read 7 Done Done Sent false 1013 Acked Read 7 Done Done Sent false 1014 Acked Flush 0 Done Done Sent false 1015 Acked Flush 0 Done Done Sent false 1016 Acked Write 7 Done Done Sent false 1017 Acked Read 3 Done Done Sent false 1018 Acked Write 4 Done Done Sent false 1019 Acked Read 3 Done Done Sent false 1020 Acked Write 10 Done Done Sent false 1021 Acked Flush 0 Done Done Sent false 1022 Acked Write 6 Done Done Sent false 1023 Acked Read 5 Done Done Sent false 1024 Acked Read 2 Done Done Sent false 1025 Acked Write 4 Done Done Sent false 1026 Acked Write 7 Done Done Sent false 1027 Acked Read 7 Done Done Sent false 1028 Acked Read 3 Done Done Sent false 1029 Acked Read 2 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 24 24 Done 26 26 2 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1003)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:42:35.348935057Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1197} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:35.349112912Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1197,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 15 15 15 26 26 26 F F F 1 020-039 16 16 14 29 29 23 T T F <--- 2 040-059 14 14 14 23 23 23 F F F 3 060-079 12 12 12 18 18 18 F F F 4 080-099 8 8 8 15 15 15 F F F 5 100-119 16 16 15 32 32 26 F F F <--- 6 120-139 15 15 15 26 26 26 T T F <--- 7 140-159 16 16 16 27 27 27 T T F <--- 8 160-179 16 16 14 32 32 24 F F F <--- 9 180-199 7 7 7 12 12 12 F F F 10 200-219 16 16 15 32 32 26 F F F <--- 11 220-239 15 15 15 26 26 26 F F F 12 240-259 10 10 10 17 17 17 F F F 13 260-279 10 10 10 17 17 17 F F F 14 280-299 14 14 14 24 24 24 F F F 15 300-319 8 8 8 15 15 15 F F F 16 320-339 8 8 8 15 15 15 F F F 17 340-359 10 10 10 17 17 17 F F F 18 360-379 14 14 14 25 25 25 F F F 19 380-399 14 14 14 25 25 25 F F F 20 400-419 15 15 15 26 26 26 F F F 21 420-439 10 10 10 17 17 17 F F F 22 440-459 10 10 10 17 17 17 F F F 23 460-479 12 12 12 20 20 20 F F F 24 480-499 15 15 15 26 26 26 F F F 25 500-519 16 16 13 30 30 22 F F F <--- 26 520-539 16 16 14 28 28 24 F F T <--- 27 540-559 16 16 14 29 29 24 T T F <--- 28 560-579 15 15 15 26 26 26 F F F 29 580-599 7 7 7 12 12 12 F F F Max gen: 16, Max flush: 32 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 8 Verifying data now No change after verify Loop: 8 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 15 15 15 26 26 26 F F F 1 020-039 17 17 17 33 33 33 F F F 2 040-059 14 14 14 23 23 23 F F F 3 060-079 12 12 12 18 18 18 F F F 4 080-099 8 8 8 15 15 15 F F F 5 100-119 16 16 16 32 32 32 F F F 6 120-139 17 17 17 33 33 33 F F F 7 140-159 17 17 17 33 33 33 F F F 8 160-179 16 16 16 32 32 32 F F F 9 180-199 7 7 7 12 12 12 F F F 10 200-219 16 16 16 32 32 32 F F F 11 220-239 15 15 15 26 26 26 F F F 12 240-259 10 10 10 17 17 17 F F F 13 260-279 10 10 10 17 17 17 F F F 14 280-299 14 14 14 24 24 24 F F F 15 300-319 8 8 8 15 15 15 F F F 16 320-339 8 8 8 15 15 15 F F F 17 340-359 10 10 10 17 17 17 F F F 18 360-379 14 14 14 25 25 25 F F F 19 380-399 14 14 14 25 25 25 F F F 20 400-419 15 15 15 26 26 26 F F F 21 420-439 10 10 10 17 17 17 F F F 22 440-459 10 10 10 17 17 17 F F F 23 460-479 12 12 12 20 20 20 F F F 24 480-499 15 15 15 26 26 26 F F F 25 500-519 16 16 16 30 30 30 F F F 26 520-539 16 16 16 28 28 28 F F F 27 540-559 17 17 17 33 33 33 F F F 28 560-579 15 15 15 26 26 26 F F F 29 580-599 7 7 7 12 12 12 F F F Max gen: 17, Max flush: 33 Begin loop 9 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:39.58532197Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1205,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:39.593281611Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1205,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:39.593572342Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1205} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:39.593597492Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1205} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:39.593605751Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1205} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:39.593617141Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1205} {"msg":"Crucible 0d32cce1-80c8-4298-bb51-8ce1eee3ede0 has session id: ff503a96-0f75-4ac6-b4ad-c568631796a3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:39.593959821Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1205,"session_id":"ff503a96-0f75-4ac6-b4ad-c568631796a3"} {"msg":"Upstairs opts: Upstairs UUID: 0d32cce1-80c8-4298-bb51-8ce1eee3ede0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:39.594025029Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1205,"session_id":"ff503a96-0f75-4ac6-b4ad-c568631796a3"} {"msg":"Crucible stats registered with UUID: 0d32cce1-80c8-4298-bb51-8ce1eee3ede0","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:39.594036689Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1205,"session_id":"ff503a96-0f75-4ac6-b4ad-c568631796a3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:42:39.594046019Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1205,"session_id":"ff503a96-0f75-4ac6-b4ad-c568631796a3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:39.594202084Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1205,"":"io task","client":"2","":"downstairs","session_id":"ff503a96-0f75-4ac6-b4ad-c568631796a3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:39.594234153Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1205,"":"io task","client":"0","":"downstairs","session_id":"ff503a96-0f75-4ac6-b4ad-c568631796a3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:39.594245163Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1205,"":"io task","client":"1","":"downstairs","session_id":"ff503a96-0f75-4ac6-b4ad-c568631796a3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:39.594386709Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1205,"task":"crutest"} {"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:39.594405178Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1205} {"msg":"0d32cce1-80c8-4298-bb51-8ce1eee3ede0 active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:39.594428338Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1205,"session_id":"ff503a96-0f75-4ac6-b4ad-c568631796a3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:39.594437717Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1205,"client":"0","":"downstairs","session_id":"ff503a96-0f75-4ac6-b4ad-c568631796a3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:39.594447457Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1205,"client":"1","":"downstairs","session_id":"ff503a96-0f75-4ac6-b4ad-c568631796a3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:39.594456997Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1205,"client":"2","":"downstairs","session_id":"ff503a96-0f75-4ac6-b4ad-c568631796a3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:39.594466506Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1205,"":"io task","client":"2","":"downstairs","session_id":"ff503a96-0f75-4ac6-b4ad-c568631796a3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:39.594476336Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1205,"":"io task","client":"0","":"downstairs","session_id":"ff503a96-0f75-4ac6-b4ad-c568631796a3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:39.594501025Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1205,"":"io task","client":"1","":"downstairs","session_id":"ff503a96-0f75-4ac6-b4ad-c568631796a3"} {"msg":"ds_connection connected from Ok(127.0.0.1:37608)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:39.594521605Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1205,"":"io task","client":"2","":"downstairs","session_id":"ff503a96-0f75-4ac6-b4ad-c568631796a3"} {"msg":"ds_connection connected from Ok(127.0.0.1:40479)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:39.594532235Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1205,"":"io task","client":"1","":"downstairs","session_id":"ff503a96-0f75-4ac6-b4ad-c568631796a3"} {"msg":"ds_connection connected from Ok(127.0.0.1:39033)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:39.594654941Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1205,"":"io task","client":"0","":"downstairs","session_id":"ff503a96-0f75-4ac6-b4ad-c568631796a3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:39.595199445Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1205,"client":"2","":"downstairs","session_id":"ff503a96-0f75-4ac6-b4ad-c568631796a3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:39.595242674Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1205,"client":"1","":"downstairs","session_id":"ff503a96-0f75-4ac6-b4ad-c568631796a3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:39.595329942Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1205,"client":"0","":"downstairs","session_id":"ff503a96-0f75-4ac6-b4ad-c568631796a3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:39.595514116Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1205,"session_id":"ff503a96-0f75-4ac6-b4ad-c568631796a3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:39.595563095Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1205,"session_id":"ff503a96-0f75-4ac6-b4ad-c568631796a3"} {"msg":"[0]R flush_numbers[0..12]: [26, 33, 23, 18, 15, 32, 33, 33, 32, 12, 32, 26]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:39.595613663Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1205,"":"downstairs","":"downstairs","session_id":"ff503a96-0f75-4ac6-b4ad-c568631796a3"} {"msg":"[0]R generation[0..12]: [15, 17, 14, 12, 8, 16, 17, 17, 16, 7, 16, 15]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:39.595624523Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1205,"":"downstairs","":"downstairs","session_id":"ff503a96-0f75-4ac6-b4ad-c568631796a3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:39.595635493Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1205,"":"downstairs","":"downstairs","session_id":"ff503a96-0f75-4ac6-b4ad-c568631796a3"} {"msg":"[1]R flush_numbers[0..12]: [26, 33, 23, 18, 15, 32, 33, 33, 32, 12, 32, 26]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:39.595645952Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1205,"":"downstairs","":"downstairs","session_id":"ff503a96-0f75-4ac6-b4ad-c568631796a3"} {"msg":"[1]R generation[0..12]: [15, 17, 14, 12, 8, 16, 17, 17, 16, 7, 16, 15]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:39.595656222Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1205,"":"downstairs","":"downstairs","session_id":"ff503a96-0f75-4ac6-b4ad-c568631796a3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:39.595666972Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1205,"":"downstairs","":"downstairs","session_id":"ff503a96-0f75-4ac6-b4ad-c568631796a3"} {"msg":"[2]R flush_numbers[0..12]: [26, 33, 23, 18, 15, 32, 33, 33, 32, 12, 32, 26]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:39.595677312Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1205,"":"downstairs","":"downstairs","session_id":"ff503a96-0f75-4ac6-b4ad-c568631796a3"} {"msg":"[2]R generation[0..12]: [15, 17, 14, 12, 8, 16, 17, 17, 16, 7, 16, 15]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:39.595687571Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1205,"":"downstairs","":"downstairs","session_id":"ff503a96-0f75-4ac6-b4ad-c568631796a3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:39.595698521Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1205,"":"downstairs","":"downstairs","session_id":"ff503a96-0f75-4ac6-b4ad-c568631796a3"} {"msg":"Max found gen is 18","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:39.595707891Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1205,"":"downstairs","":"downstairs","session_id":"ff503a96-0f75-4ac6-b4ad-c568631796a3"} {"msg":"Generation requested: 18 >= found:18","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:39.59571765Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1205,"":"downstairs","":"downstairs","session_id":"ff503a96-0f75-4ac6-b4ad-c568631796a3"} {"msg":"Next flush: 34","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:39.595727Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1205,"":"downstairs","":"downstairs","session_id":"ff503a96-0f75-4ac6-b4ad-c568631796a3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:39.59573672Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1205,"":"downstairs","":"downstairs","session_id":"ff503a96-0f75-4ac6-b4ad-c568631796a3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:39.59574651Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1205,"session_id":"ff503a96-0f75-4ac6-b4ad-c568631796a3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:39.595756449Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1205,"client":"0","":"downstairs","session_id":"ff503a96-0f75-4ac6-b4ad-c568631796a3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:39.595766309Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1205,"client":"1","":"downstairs","session_id":"ff503a96-0f75-4ac6-b4ad-c568631796a3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:39.595776039Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1205,"client":"2","":"downstairs","session_id":"ff503a96-0f75-4ac6-b4ad-c568631796a3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:39.595786088Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1205,"session_id":"ff503a96-0f75-4ac6-b4ad-c568631796a3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:39.595795238Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1205,"session_id":"ff503a96-0f75-4ac6-b4ad-c568631796a3"} {"msg":"0d32cce1-80c8-4298-bb51-8ce1eee3ede0 is now active with session: ff503a96-0f75-4ac6-b4ad-c568631796a3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:39.595805038Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1205,"session_id":"ff503a96-0f75-4ac6-b4ad-c568631796a3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:39.595813648Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1205,"session_id":"ff503a96-0f75-4ac6-b4ad-c568631796a3"} {"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:39.595822617Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1205} {"msg":"Activated sub_volume 0d32cce1-80c8-4298-bb51-8ce1eee3ede0","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:39.595830567Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1205} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:39.595838527Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1205,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:39.596153828Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1205,"task":"crutest"} 01/30 Flush 02/30 Write block 495 len 40960 data: 2 2 2 2 2 2 2 2 2 2 03/30 Write block 519 len 12288 data: 2 4 5 04/30 Write block 143 len 16384 data: 3 3 3 4 05/30 Write block 213 len 20480 data: 4 3 3 5 5 06/30 Read block 125 len 4096 07/30 Read block 399 len 24576 08/30 Read block 249 len 40960 09/30 Read block 258 len 16384 10/30 Write block 557 len 12288 data: 4 4 4 11/30 Read block 442 len 4096 12/30 Write block 270 len 20480 data: 2 2 2 3 3 13/30 Write block 165 len 36864 data: 4 3 3 3 4 4 4 3 4 14/30 Write block 246 len 40960 data: 2 2 3 3 4 2 2 3 3 3 15/30 Flush 16/30 Write block 365 len 12288 data: 4 4 4 17/30 Read block 542 len 4096 18/30 Write block 379 len 40960 data: 5 5 4 4 3 3 2 2 2 2 19/30 Write block 476 len 8192 data: 2 2 20/30 Read block 425 len 12288 21/30 Read block 267 len 24576 22/30 Read block 65 len 36864 23/30 Read block 539 len 8192 24/30 Read block 40 len 8192 25/30 Read block 28 len 8192 26/30 Flush 27/30 Write block 368 len 12288 data: 3 3 2 28/30 Read block 549 len 36864 29/30 Write block 304 len 40960 data: 2 2 2 2 3 3 3 3 3 3 30/30 Write block 594 len 24576 data: 2 2 2 2 2 2 ---------------------------------------------------------------- Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 10 Done Done Done false 1002 Acked Write 3 Done Done Done false 1003 Acked Write 4 Done Sent Done false 1004 Acked Write 5 Done Sent Done false 1005 Acked Read 1 Done Sent Done false 1006 Acked Read 6 Done Sent Done false 1007 Acked Read 10 Done Sent Done false 1008 Acked Read 4 Done Sent Done false 1009 Acked Write 3 Done Sent Done false 1010 Acked Read 1 Done Sent Done false 1011 Acked Write 5 Done Sent Done false 1012 Acked Write 9 Done Sent Done false 1013 Acked Write 10 Done Sent Done false 1014 Acked Flush 0 Done Sent Done false 1015 Acked Write 3 Done Sent Done false 1016 Acked Read 1 Done Sent Done false 1017 Acked Write 10 Done Sent Done false 1018 Acked Write 2 Done Sent Done false 1019 Acked Read 3 Done Sent Done false 1020 Acked Read 6 Done Sent Done false 1021 Acked Read 9 Done Sent Done false 1022 Acked Read 2 Done Sent Done false 1023 Acked Read 2 Done Sent Done false 1024 Acked Read 2 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 3 Done Sent Done false 1027 Acked Read 9 Done Sent Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 27 2 31 Done 27 2 27 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1000)) Some(JobId(1025)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:42:39.604347491Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1205} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:39.604545796Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1205,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 15 15 15 26 26 26 F F F 1 020-039 17 17 17 33 33 33 F F F 2 040-059 14 14 14 23 23 23 F F F 3 060-079 12 12 12 18 18 18 F F F 4 080-099 8 8 8 15 15 15 F F F 5 100-119 16 16 16 32 32 32 F F F 6 120-139 17 17 17 33 33 33 F F F 7 140-159 18 17 18 35 33 35 F F F <--- 8 160-179 18 16 18 35 32 35 F F F <--- 9 180-199 7 7 7 12 12 12 F F F 10 200-219 18 16 18 35 32 35 F F F <--- 11 220-239 15 15 15 26 26 26 F F F 12 240-259 18 10 18 35 17 35 F F F <--- 13 260-279 18 10 18 35 17 35 F F F <--- 14 280-299 14 14 14 24 24 24 F F F 15 300-319 8 8 8 15 15 15 T F T <--- 16 320-339 8 8 8 15 15 15 F F F 17 340-359 10 10 10 17 17 17 F F F 18 360-379 18 14 18 36 25 36 T F T <--- 19 380-399 18 14 18 36 25 36 F F F <--- 20 400-419 15 15 15 26 26 26 F F F 21 420-439 10 10 10 17 17 17 F F F 22 440-459 10 10 10 17 17 17 F F F 23 460-479 18 12 18 36 20 36 F F F <--- 24 480-499 18 15 18 35 26 35 F T F <--- 25 500-519 18 16 18 35 30 35 F T F <--- 26 520-539 18 16 18 35 28 35 F T F <--- 27 540-559 18 17 18 35 33 35 F F F <--- 28 560-579 15 15 15 26 26 26 F F F 29 580-599 7 7 7 12 12 12 T F T <--- Max gen: 18, Max flush: 36 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 9 Verifying data now No change after verify Loop: 9 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 15 15 15 26 26 26 F F F 1 020-039 17 17 17 33 33 33 F F F 2 040-059 14 14 14 23 23 23 F F F 3 060-079 12 12 12 18 18 18 F F F 4 080-099 8 8 8 15 15 15 F F F 5 100-119 16 16 16 32 32 32 F F F 6 120-139 17 17 17 33 33 33 F F F 7 140-159 18 18 18 35 35 35 F F F 8 160-179 18 18 18 35 35 35 F F F 9 180-199 7 7 7 12 12 12 F F F 10 200-219 18 18 18 35 35 35 F F F 11 220-239 15 15 15 26 26 26 F F F 12 240-259 18 18 18 35 35 35 F F F 13 260-279 18 18 18 35 35 35 F F F 14 280-299 14 14 14 24 24 24 F F F 15 300-319 19 19 19 37 37 37 F F F 16 320-339 8 8 8 15 15 15 F F F 17 340-359 10 10 10 17 17 17 F F F 18 360-379 19 19 19 37 37 37 F F F 19 380-399 18 18 18 36 36 36 F F F 20 400-419 15 15 15 26 26 26 F F F 21 420-439 10 10 10 17 17 17 F F F 22 440-459 10 10 10 17 17 17 F F F 23 460-479 18 18 18 36 36 36 F F F 24 480-499 18 18 18 35 35 35 F F F 25 500-519 18 18 18 35 35 35 F F F 26 520-539 18 18 18 35 35 35 F F F 27 540-559 18 18 18 35 35 35 F F F 28 560-579 15 15 15 26 26 26 F F F 29 580-599 19 19 19 37 37 37 F F F Max gen: 19, Max flush: 37 Begin loop 10 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:43.84848809Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1213,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:43.856617116Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1213,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:43.856919527Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1213} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:43.856945416Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1213} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:43.856953856Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1213} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:43.856965326Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1213} {"msg":"Crucible 76c1e691-e0fa-44c5-932e-07997b1cecb3 has session id: ec979d86-2273-4ae3-8d41-a7eb8d869852","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:43.857292996Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1213,"session_id":"ec979d86-2273-4ae3-8d41-a7eb8d869852"} {"msg":"Upstairs opts: Upstairs UUID: 76c1e691-e0fa-44c5-932e-07997b1cecb3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:43.857321426Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1213,"session_id":"ec979d86-2273-4ae3-8d41-a7eb8d869852"} {"msg":"Crucible stats registered with UUID: 76c1e691-e0fa-44c5-932e-07997b1cecb3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:43.857332305Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1213,"session_id":"ec979d86-2273-4ae3-8d41-a7eb8d869852"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:42:43.857341585Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1213,"session_id":"ec979d86-2273-4ae3-8d41-a7eb8d869852"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:43.857473691Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1213,"":"io task","client":"0","":"downstairs","session_id":"ec979d86-2273-4ae3-8d41-a7eb8d869852"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:43.857492421Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1213,"":"io task","client":"2","":"downstairs","session_id":"ec979d86-2273-4ae3-8d41-a7eb8d869852"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:43.85750246Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1213,"":"io task","client":"1","":"downstairs","session_id":"ec979d86-2273-4ae3-8d41-a7eb8d869852"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:43.857648036Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1213,"task":"crutest"} {"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:43.857659646Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1213} {"msg":"76c1e691-e0fa-44c5-932e-07997b1cecb3 active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:43.857675545Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1213,"session_id":"ec979d86-2273-4ae3-8d41-a7eb8d869852"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:43.857684305Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1213,"client":"0","":"downstairs","session_id":"ec979d86-2273-4ae3-8d41-a7eb8d869852"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:43.857693915Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1213,"client":"1","":"downstairs","session_id":"ec979d86-2273-4ae3-8d41-a7eb8d869852"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:43.857703395Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1213,"client":"2","":"downstairs","session_id":"ec979d86-2273-4ae3-8d41-a7eb8d869852"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:43.857712814Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1213,"":"io task","client":"2","":"downstairs","session_id":"ec979d86-2273-4ae3-8d41-a7eb8d869852"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:43.857722574Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1213,"":"io task","client":"0","":"downstairs","session_id":"ec979d86-2273-4ae3-8d41-a7eb8d869852"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:43.857734884Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1213,"":"io task","client":"1","":"downstairs","session_id":"ec979d86-2273-4ae3-8d41-a7eb8d869852"} {"msg":"ds_connection connected from Ok(127.0.0.1:58480)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:43.857812611Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1213,"":"io task","client":"2","":"downstairs","session_id":"ec979d86-2273-4ae3-8d41-a7eb8d869852"} {"msg":"ds_connection connected from Ok(127.0.0.1:60116)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:43.857946947Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1213,"":"io task","client":"0","":"downstairs","session_id":"ec979d86-2273-4ae3-8d41-a7eb8d869852"} {"msg":"ds_connection connected from Ok(127.0.0.1:60878)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:43.857966047Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1213,"":"io task","client":"1","":"downstairs","session_id":"ec979d86-2273-4ae3-8d41-a7eb8d869852"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:43.858479282Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1213,"client":"0","":"downstairs","session_id":"ec979d86-2273-4ae3-8d41-a7eb8d869852"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:43.858496422Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1213,"client":"1","":"downstairs","session_id":"ec979d86-2273-4ae3-8d41-a7eb8d869852"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:43.85856824Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1213,"client":"2","":"downstairs","session_id":"ec979d86-2273-4ae3-8d41-a7eb8d869852"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:43.858834002Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1213,"session_id":"ec979d86-2273-4ae3-8d41-a7eb8d869852"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:43.858843352Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1213,"session_id":"ec979d86-2273-4ae3-8d41-a7eb8d869852"} {"msg":"[0]R flush_numbers[0..12]: [26, 33, 23, 18, 15, 32, 33, 35, 35, 12, 35, 26]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:43.85888896Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1213,"":"downstairs","":"downstairs","session_id":"ec979d86-2273-4ae3-8d41-a7eb8d869852"} {"msg":"[0]R generation[0..12]: [15, 17, 14, 12, 8, 16, 17, 18, 18, 7, 18, 15]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:43.85889985Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1213,"":"downstairs","":"downstairs","session_id":"ec979d86-2273-4ae3-8d41-a7eb8d869852"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:43.85891077Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1213,"":"downstairs","":"downstairs","session_id":"ec979d86-2273-4ae3-8d41-a7eb8d869852"} {"msg":"[1]R flush_numbers[0..12]: [26, 33, 23, 18, 15, 32, 33, 35, 35, 12, 35, 26]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:43.858921079Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1213,"":"downstairs","":"downstairs","session_id":"ec979d86-2273-4ae3-8d41-a7eb8d869852"} {"msg":"[1]R generation[0..12]: [15, 17, 14, 12, 8, 16, 17, 18, 18, 7, 18, 15]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:43.858931319Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1213,"":"downstairs","":"downstairs","session_id":"ec979d86-2273-4ae3-8d41-a7eb8d869852"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:43.858944519Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1213,"":"downstairs","":"downstairs","session_id":"ec979d86-2273-4ae3-8d41-a7eb8d869852"} {"msg":"[2]R flush_numbers[0..12]: [26, 33, 23, 18, 15, 32, 33, 35, 35, 12, 35, 26]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:43.858954748Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1213,"":"downstairs","":"downstairs","session_id":"ec979d86-2273-4ae3-8d41-a7eb8d869852"} {"msg":"[2]R generation[0..12]: [15, 17, 14, 12, 8, 16, 17, 18, 18, 7, 18, 15]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:43.858964838Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1213,"":"downstairs","":"downstairs","session_id":"ec979d86-2273-4ae3-8d41-a7eb8d869852"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:43.858975608Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1213,"":"downstairs","":"downstairs","session_id":"ec979d86-2273-4ae3-8d41-a7eb8d869852"} {"msg":"Max found gen is 20","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:43.858989967Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1213,"":"downstairs","":"downstairs","session_id":"ec979d86-2273-4ae3-8d41-a7eb8d869852"} {"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:43.858999557Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1213,"":"downstairs","":"downstairs","session_id":"ec979d86-2273-4ae3-8d41-a7eb8d869852"} {"msg":"Next flush: 38","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:43.859008847Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1213,"":"downstairs","":"downstairs","session_id":"ec979d86-2273-4ae3-8d41-a7eb8d869852"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:43.859018157Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1213,"":"downstairs","":"downstairs","session_id":"ec979d86-2273-4ae3-8d41-a7eb8d869852"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:43.859027876Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1213,"session_id":"ec979d86-2273-4ae3-8d41-a7eb8d869852"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:43.859036516Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1213,"client":"0","":"downstairs","session_id":"ec979d86-2273-4ae3-8d41-a7eb8d869852"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:43.859046056Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1213,"client":"1","":"downstairs","session_id":"ec979d86-2273-4ae3-8d41-a7eb8d869852"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:43.859055566Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1213,"client":"2","":"downstairs","session_id":"ec979d86-2273-4ae3-8d41-a7eb8d869852"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:43.859065215Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1213,"session_id":"ec979d86-2273-4ae3-8d41-a7eb8d869852"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:43.859074205Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1213,"session_id":"ec979d86-2273-4ae3-8d41-a7eb8d869852"} {"msg":"76c1e691-e0fa-44c5-932e-07997b1cecb3 is now active with session: ec979d86-2273-4ae3-8d41-a7eb8d869852","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:43.859084195Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1213,"session_id":"ec979d86-2273-4ae3-8d41-a7eb8d869852"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:43.859094224Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1213,"session_id":"ec979d86-2273-4ae3-8d41-a7eb8d869852"} {"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:43.859103264Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1213} {"msg":"Activated sub_volume 76c1e691-e0fa-44c5-932e-07997b1cecb3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:43.859111574Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1213} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:43.859119744Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1213,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:43.859476523Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1213,"task":"crutest"} 01/30 Write block 343 len 32768 data: 3 2 2 2 2 2 2 2 02/30 Flush 03/30 Write block 217 len 40960 data: 6 3 3 2 2 3 5 5 5 5 04/30 Read block 2 len 12288 05/30 Write block 484 len 20480 data: 2 3 3 4 3 06/30 Read block 392 len 4096 07/30 Read block 525 len 36864 08/30 Read block 265 len 4096 09/30 Write block 590 len 28672 data: 4 4 3 2 3 3 3 10/30 Read block 92 len 40960 11/30 Read block 361 len 4096 12/30 Read block 265 len 32768 13/30 Read block 273 len 4096 14/30 Read block 118 len 20480 15/30 Read block 193 len 4096 16/30 Read block 228 len 32768 17/30 Read block 194 len 24576 18/30 Read block 571 len 12288 19/30 Read block 243 len 40960 20/30 Write block 253 len 16384 data: 4 4 4 3 21/30 Write block 495 len 40960 data: 3 3 3 3 3 3 3 3 3 3 22/30 Read block 495 len 20480 23/30 Read block 549 len 40960 24/30 Write block 201 len 24576 data: 4 4 4 4 3 4 25/30 Read block 167 len 8192 26/30 Read block 328 len 20480 27/30 Write block 587 len 8192 data: 4 4 28/30 Write block 483 len 4096 data: 2 29/30 Write block 532 len 16384 data: 4 4 4 4 30/30 Read block 422 len 8192 ---------------------------------------------------------------- Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Flush 0 Done Done Sent false 1002 Acked Write 10 Done Done Sent false 1003 Acked Read 3 Done Done Sent false 1004 Acked Write 5 Done Done Sent false 1005 Acked Read 1 Done Done Sent false 1006 Acked Read 9 Done Done Sent false 1007 Acked Read 1 Done Done Sent false 1008 Acked Write 7 Done Done Sent false 1009 Acked Read 10 Done Done Sent false 1010 Acked Read 1 Done Done Sent false 1011 Acked Read 8 Done Done Sent false 1012 Acked Read 1 Done Done Sent false 1013 Acked Read 5 Done Done Sent false 1014 Acked Read 1 Done Done Sent false 1015 Acked Read 8 Done Done Sent false 1016 Acked Read 6 Done Done Sent false 1017 Acked Read 3 Done Done Sent false 1018 Acked Read 10 Done Done Sent false 1019 Acked Write 4 Done Done Sent false 1020 Acked Write 10 Done Done Sent false 1021 Acked Read 5 Done Done Sent false 1022 Acked Read 10 Done Done Sent false 1023 Acked Write 6 Done Done Sent false 1024 Acked Read 2 Done Done Sent false 1025 Acked Read 5 Done Done Sent false 1026 Acked Write 2 Done Done Sent false 1027 Acked Write 1 Done Done Sent false 1028 Acked Write 4 Done Done Sent false 1029 Acked Read 2 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 29 29 Done 30 30 1 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1001)) Some(JobId(1001)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:42:43.868220471Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1213} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:43.868416535Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1213,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 15 15 15 26 26 26 F F F 1 020-039 17 17 17 33 33 33 F F F 2 040-059 14 14 14 23 23 23 F F F 3 060-079 12 12 12 18 18 18 F F F 4 080-099 8 8 8 15 15 15 F F F 5 100-119 16 16 16 32 32 32 F F F 6 120-139 17 17 17 33 33 33 F F F 7 140-159 18 18 18 35 35 35 F F F 8 160-179 18 18 18 35 35 35 F F F 9 180-199 7 7 7 12 12 12 F F F 10 200-219 18 18 18 35 35 35 T T F <--- 11 220-239 15 15 15 26 26 26 T T F <--- 12 240-259 18 18 18 35 35 35 T T F <--- 13 260-279 18 18 18 35 35 35 F F F 14 280-299 14 14 14 24 24 24 F F F 15 300-319 19 19 19 37 37 37 F F F 16 320-339 8 8 8 15 15 15 F F F 17 340-359 20 20 10 38 38 17 F F T <--- 18 360-379 19 19 19 37 37 37 F F F 19 380-399 18 18 18 36 36 36 F F F 20 400-419 15 15 15 26 26 26 F F F 21 420-439 10 10 10 17 17 17 F F F 22 440-459 10 10 10 17 17 17 F F F 23 460-479 18 18 18 36 36 36 F F F 24 480-499 18 18 18 35 35 35 T T F <--- 25 500-519 18 18 18 35 35 35 T T F <--- 26 520-539 18 18 18 35 35 35 T T F <--- 27 540-559 18 18 18 35 35 35 F F F 28 560-579 15 15 15 26 26 26 F F F 29 580-599 19 19 19 37 37 37 T T F <--- Max gen: 20, Max flush: 38 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 10 Verifying data now No change after verify Loop: 10 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 15 15 15 26 26 26 F F F 1 020-039 17 17 17 33 33 33 F F F 2 040-059 14 14 14 23 23 23 F F F 3 060-079 12 12 12 18 18 18 F F F 4 080-099 8 8 8 15 15 15 F F F 5 100-119 16 16 16 32 32 32 F F F 6 120-139 17 17 17 33 33 33 F F F 7 140-159 18 18 18 35 35 35 F F F 8 160-179 18 18 18 35 35 35 F F F 9 180-199 7 7 7 12 12 12 F F F 10 200-219 21 21 21 39 39 39 F F F 11 220-239 21 21 21 39 39 39 F F F 12 240-259 21 21 21 39 39 39 F F F 13 260-279 18 18 18 35 35 35 F F F 14 280-299 14 14 14 24 24 24 F F F 15 300-319 19 19 19 37 37 37 F F F 16 320-339 8 8 8 15 15 15 F F F 17 340-359 20 20 20 38 38 38 F F F 18 360-379 19 19 19 37 37 37 F F F 19 380-399 18 18 18 36 36 36 F F F 20 400-419 15 15 15 26 26 26 F F F 21 420-439 10 10 10 17 17 17 F F F 22 440-459 10 10 10 17 17 17 F F F 23 460-479 18 18 18 36 36 36 F F F 24 480-499 21 21 21 39 39 39 F F F 25 500-519 21 21 21 39 39 39 F F F 26 520-539 21 21 21 39 39 39 F F F 27 540-559 18 18 18 35 35 35 F F F 28 560-579 15 15 15 26 26 26 F F F 29 580-599 21 21 21 39 39 39 F F F Max gen: 21, Max flush: 39 Begin loop 11 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:48.106954776Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1221,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:48.115056523Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1221,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:48.115354854Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1221} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:48.115383903Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1221} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:48.115392273Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1221} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:48.115403823Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1221} {"msg":"Crucible b4a0edc7-dbff-4ad0-b254-af28bf394774 has session id: d2138791-20a8-4170-a9d0-a3c32be33ef5","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:48.115720504Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1221,"session_id":"d2138791-20a8-4170-a9d0-a3c32be33ef5"} {"msg":"Upstairs opts: Upstairs UUID: b4a0edc7-dbff-4ad0-b254-af28bf394774, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:48.115749463Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1221,"session_id":"d2138791-20a8-4170-a9d0-a3c32be33ef5"} {"msg":"Crucible stats registered with UUID: b4a0edc7-dbff-4ad0-b254-af28bf394774","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:48.115760292Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1221,"session_id":"d2138791-20a8-4170-a9d0-a3c32be33ef5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:42:48.115769502Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1221,"session_id":"d2138791-20a8-4170-a9d0-a3c32be33ef5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:48.115914158Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1221,"":"io task","client":"2","":"downstairs","session_id":"d2138791-20a8-4170-a9d0-a3c32be33ef5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:48.115945567Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1221,"":"io task","client":"0","":"downstairs","session_id":"d2138791-20a8-4170-a9d0-a3c32be33ef5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:48.115956597Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1221,"":"io task","client":"1","":"downstairs","session_id":"d2138791-20a8-4170-a9d0-a3c32be33ef5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:48.116101593Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1221,"task":"crutest"} {"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:48.116112642Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1221} {"msg":"b4a0edc7-dbff-4ad0-b254-af28bf394774 active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:48.116123522Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1221,"session_id":"d2138791-20a8-4170-a9d0-a3c32be33ef5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:48.116132272Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1221,"client":"0","":"downstairs","session_id":"d2138791-20a8-4170-a9d0-a3c32be33ef5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:48.116141851Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1221,"client":"1","":"downstairs","session_id":"d2138791-20a8-4170-a9d0-a3c32be33ef5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:48.116151511Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1221,"client":"2","":"downstairs","session_id":"d2138791-20a8-4170-a9d0-a3c32be33ef5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:48.116160941Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1221,"":"io task","client":"2","":"downstairs","session_id":"d2138791-20a8-4170-a9d0-a3c32be33ef5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:48.116170861Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1221,"":"io task","client":"0","":"downstairs","session_id":"d2138791-20a8-4170-a9d0-a3c32be33ef5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:48.11618055Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1221,"":"io task","client":"1","":"downstairs","session_id":"d2138791-20a8-4170-a9d0-a3c32be33ef5"} {"msg":"ds_connection connected from Ok(127.0.0.1:63811)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:48.116354545Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1221,"":"io task","client":"1","":"downstairs","session_id":"d2138791-20a8-4170-a9d0-a3c32be33ef5"} {"msg":"ds_connection connected from Ok(127.0.0.1:36217)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:48.116374695Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1221,"":"io task","client":"2","":"downstairs","session_id":"d2138791-20a8-4170-a9d0-a3c32be33ef5"} {"msg":"ds_connection connected from Ok(127.0.0.1:51986)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:48.116385404Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1221,"":"io task","client":"0","":"downstairs","session_id":"d2138791-20a8-4170-a9d0-a3c32be33ef5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:48.117015026Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1221,"client":"2","":"downstairs","session_id":"d2138791-20a8-4170-a9d0-a3c32be33ef5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:48.117061805Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1221,"client":"1","":"downstairs","session_id":"d2138791-20a8-4170-a9d0-a3c32be33ef5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:48.117075515Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1221,"client":"0","":"downstairs","session_id":"d2138791-20a8-4170-a9d0-a3c32be33ef5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:48.117308318Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1221,"session_id":"d2138791-20a8-4170-a9d0-a3c32be33ef5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:48.117347167Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1221,"session_id":"d2138791-20a8-4170-a9d0-a3c32be33ef5"} {"msg":"[0]R flush_numbers[0..12]: [26, 33, 23, 18, 15, 32, 33, 35, 35, 12, 39, 39]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:48.117396315Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1221,"":"downstairs","":"downstairs","session_id":"d2138791-20a8-4170-a9d0-a3c32be33ef5"} {"msg":"[0]R generation[0..12]: [15, 17, 14, 12, 8, 16, 17, 18, 18, 7, 21, 21]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:48.117407355Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1221,"":"downstairs","":"downstairs","session_id":"d2138791-20a8-4170-a9d0-a3c32be33ef5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:48.117418345Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1221,"":"downstairs","":"downstairs","session_id":"d2138791-20a8-4170-a9d0-a3c32be33ef5"} {"msg":"[1]R flush_numbers[0..12]: [26, 33, 23, 18, 15, 32, 33, 35, 35, 12, 39, 39]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:48.117428694Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1221,"":"downstairs","":"downstairs","session_id":"d2138791-20a8-4170-a9d0-a3c32be33ef5"} {"msg":"[1]R generation[0..12]: [15, 17, 14, 12, 8, 16, 17, 18, 18, 7, 21, 21]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:48.117439034Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1221,"":"downstairs","":"downstairs","session_id":"d2138791-20a8-4170-a9d0-a3c32be33ef5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:48.117449754Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1221,"":"downstairs","":"downstairs","session_id":"d2138791-20a8-4170-a9d0-a3c32be33ef5"} {"msg":"[2]R flush_numbers[0..12]: [26, 33, 23, 18, 15, 32, 33, 35, 35, 12, 39, 39]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:48.117460083Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1221,"":"downstairs","":"downstairs","session_id":"d2138791-20a8-4170-a9d0-a3c32be33ef5"} {"msg":"[2]R generation[0..12]: [15, 17, 14, 12, 8, 16, 17, 18, 18, 7, 21, 21]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:48.117470313Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1221,"":"downstairs","":"downstairs","session_id":"d2138791-20a8-4170-a9d0-a3c32be33ef5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:48.117481133Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1221,"":"downstairs","":"downstairs","session_id":"d2138791-20a8-4170-a9d0-a3c32be33ef5"} {"msg":"Max found gen is 22","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:48.117490513Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1221,"":"downstairs","":"downstairs","session_id":"d2138791-20a8-4170-a9d0-a3c32be33ef5"} {"msg":"Generation requested: 22 >= found:22","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:48.117500152Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1221,"":"downstairs","":"downstairs","session_id":"d2138791-20a8-4170-a9d0-a3c32be33ef5"} {"msg":"Next flush: 40","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:48.117510922Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1221,"":"downstairs","":"downstairs","session_id":"d2138791-20a8-4170-a9d0-a3c32be33ef5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:48.117520262Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1221,"":"downstairs","":"downstairs","session_id":"d2138791-20a8-4170-a9d0-a3c32be33ef5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:48.117529931Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1221,"session_id":"d2138791-20a8-4170-a9d0-a3c32be33ef5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:48.117538651Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1221,"client":"0","":"downstairs","session_id":"d2138791-20a8-4170-a9d0-a3c32be33ef5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:48.117548141Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1221,"client":"1","":"downstairs","session_id":"d2138791-20a8-4170-a9d0-a3c32be33ef5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:48.117557781Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1221,"client":"2","":"downstairs","session_id":"d2138791-20a8-4170-a9d0-a3c32be33ef5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:48.11756747Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1221,"session_id":"d2138791-20a8-4170-a9d0-a3c32be33ef5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:48.11757677Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1221,"session_id":"d2138791-20a8-4170-a9d0-a3c32be33ef5"} {"msg":"b4a0edc7-dbff-4ad0-b254-af28bf394774 is now active with session: d2138791-20a8-4170-a9d0-a3c32be33ef5","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:48.11758659Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1221,"session_id":"d2138791-20a8-4170-a9d0-a3c32be33ef5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:48.11759517Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1221,"session_id":"d2138791-20a8-4170-a9d0-a3c32be33ef5"} {"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:48.117604119Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1221} {"msg":"Activated sub_volume b4a0edc7-dbff-4ad0-b254-af28bf394774","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:48.117612259Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1221} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:48.117620369Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1221,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:48.117958969Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1221,"task":"crutest"} 01/30 Write block 312 len 32768 data: 4 4 3 4 4 2 2 2 02/30 Read block 46 len 20480 03/30 Write block 166 len 28672 data: 4 4 4 5 5 5 4 04/30 Read block 492 len 36864 05/30 Read block 484 len 28672 06/30 Flush 07/30 Write block 439 len 12288 data: 4 4 4 08/30 Write block 534 len 28672 data: 5 5 3 3 3 4 4 09/30 Read block 1 len 12288 10/30 Flush 11/30 Write block 246 len 40960 data: 3 3 4 4 5 3 3 5 5 5 12/30 Read block 271 len 4096 13/30 Read block 458 len 28672 14/30 Read block 344 len 8192 15/30 Read block 585 len 36864 16/30 Read block 422 len 32768 17/30 Flush 18/30 Write block 203 len 12288 data: 5 5 4 19/30 Write block 236 len 28672 data: 2 2 2 2 2 2 2 20/30 Read block 75 len 28672 21/30 Read block 302 len 36864 22/30 Read block 582 len 12288 23/30 Read block 182 len 36864 24/30 Write block 535 len 16384 data: 6 4 4 4 25/30 Flush 26/30 Write block 75 len 24576 data: 3 4 4 4 3 3 27/30 Read block 292 len 20480 28/30 Write block 84 len 4096 data: 2 29/30 Read block 369 len 12288 30/30 Read block 145 len 12288 ---------------------------------------------------------------- Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Sent false 1001 Acked Read 5 Done Done Sent false 1002 Acked Write 7 Done Done Sent false 1003 Acked Read 9 Done Done Sent false 1004 Acked Read 7 Done Done Sent false 1005 Acked Flush 0 Done Done Sent false 1006 Acked Write 3 Done Done Sent false 1007 Acked Write 7 Done Done Sent false 1008 Acked Read 3 Done Done Sent false 1009 Acked Flush 0 Done Done Sent false 1010 Acked Write 10 Done Done Sent false 1011 Acked Read 1 Done Done Sent false 1012 Acked Read 7 Done Done Sent false 1013 Acked Read 2 Done Done Sent false 1014 Acked Read 9 Done Done Sent false 1015 Acked Read 8 Done Done Sent false 1016 Acked Flush 0 Done Done Sent false 1017 Acked Write 3 Done Done Sent false 1018 Acked Write 7 Done Done Sent false 1019 Acked Read 7 Done Done Sent false 1020 Acked Read 9 Done Done Sent false 1021 Acked Read 3 Done Done Sent false 1022 Acked Read 9 Done Done Sent false 1023 Acked Write 4 Done Done Sent false 1024 Acked Flush 0 Done Done Sent false 1025 Acked Write 6 Done Done Sent false 1026 Acked Read 5 Done Done Sent false 1027 Acked Write 1 Done Done Sent false 1028 Acked Read 3 Done Done Sent false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 30 31 Done 30 29 0 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:42:48.127110275Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1221} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:48.12728098Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1221,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 15 15 15 26 26 26 F F F 1 020-039 17 17 17 33 33 33 F F F 2 040-059 14 14 14 23 23 23 F F F 3 060-079 12 12 12 18 18 18 T T F <--- 4 080-099 8 8 8 15 15 15 T T F <--- 5 100-119 16 16 16 32 32 32 F F F 6 120-139 17 17 17 33 33 33 F F F 7 140-159 18 18 18 35 35 35 F F F 8 160-179 22 22 18 40 40 35 F F F <--- 9 180-199 7 7 7 12 12 12 F F F 10 200-219 22 22 21 43 43 39 F F F <--- 11 220-239 22 22 21 43 43 39 F F F <--- 12 240-259 22 22 21 43 43 39 F F F <--- 13 260-279 18 18 18 35 35 35 F F F 14 280-299 14 14 14 24 24 24 F F F 15 300-319 22 22 19 40 40 37 F F F <--- 16 320-339 8 8 8 15 15 15 F F F 17 340-359 20 20 20 38 38 38 F F F 18 360-379 19 19 19 37 37 37 F F F 19 380-399 18 18 18 36 36 36 F F F 20 400-419 15 15 15 26 26 26 F F F 21 420-439 22 22 10 41 41 17 F F F <--- 22 440-459 22 22 10 41 41 17 F F F <--- 23 460-479 18 18 18 36 36 36 F F F 24 480-499 21 21 21 39 39 39 F F F 25 500-519 21 21 21 39 39 39 F F F 26 520-539 22 22 21 43 43 39 F F F <--- 27 540-559 22 22 18 41 41 35 F F F <--- 28 560-579 15 15 15 26 26 26 F F F 29 580-599 21 21 21 39 39 39 F F F Max gen: 22, Max flush: 43 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 11 Verifying data now No change after verify Loop: 11 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 15 15 15 26 26 26 F F F 1 020-039 17 17 17 33 33 33 F F F 2 040-059 14 14 14 23 23 23 F F F 3 060-079 23 23 23 44 44 44 F F F 4 080-099 23 23 23 44 44 44 F F F 5 100-119 16 16 16 32 32 32 F F F 6 120-139 17 17 17 33 33 33 F F F 7 140-159 18 18 18 35 35 35 F F F 8 160-179 22 22 22 40 40 40 F F F 9 180-199 7 7 7 12 12 12 F F F 10 200-219 22 22 22 43 43 43 F F F 11 220-239 22 22 22 43 43 43 F F F 12 240-259 22 22 22 43 43 43 F F F 13 260-279 18 18 18 35 35 35 F F F 14 280-299 14 14 14 24 24 24 F F F 15 300-319 22 22 22 40 40 40 F F F 16 320-339 8 8 8 15 15 15 F F F 17 340-359 20 20 20 38 38 38 F F F 18 360-379 19 19 19 37 37 37 F F F 19 380-399 18 18 18 36 36 36 F F F 20 400-419 15 15 15 26 26 26 F F F 21 420-439 22 22 22 41 41 41 F F F 22 440-459 22 22 22 41 41 41 F F F 23 460-479 18 18 18 36 36 36 F F F 24 480-499 21 21 21 39 39 39 F F F 25 500-519 21 21 21 39 39 39 F F F 26 520-539 22 22 22 43 43 43 F F F 27 540-559 22 22 22 41 41 41 F F F 28 560-579 15 15 15 26 26 26 F F F 29 580-599 21 21 21 39 39 39 F F F Max gen: 23, Max flush: 44 Begin loop 12 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:52.366483432Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1229,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:42:52.37348968Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1229,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:52.373544698Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1229,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:52.380348242Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1229,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:52.380630974Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1229} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:52.380654493Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1229} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:52.380663033Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1229} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:52.380674493Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1229} {"msg":"Crucible 1e32d877-17d0-4e71-bf75-f10cf94290e5 has session id: 14548c6d-7114-4a90-9fc8-b48c94ed6f71","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:52.381025692Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1229,"session_id":"14548c6d-7114-4a90-9fc8-b48c94ed6f71"} {"msg":"Upstairs opts: Upstairs UUID: 1e32d877-17d0-4e71-bf75-f10cf94290e5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:52.381049522Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1229,"session_id":"14548c6d-7114-4a90-9fc8-b48c94ed6f71"} {"msg":"Crucible stats registered with UUID: 1e32d877-17d0-4e71-bf75-f10cf94290e5","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:52.381059602Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1229,"session_id":"14548c6d-7114-4a90-9fc8-b48c94ed6f71"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:42:52.381068701Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1229,"session_id":"14548c6d-7114-4a90-9fc8-b48c94ed6f71"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:52.381208107Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1229,"":"io task","client":"2","":"downstairs","session_id":"14548c6d-7114-4a90-9fc8-b48c94ed6f71"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:52.381239396Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1229,"":"io task","client":"0","":"downstairs","session_id":"14548c6d-7114-4a90-9fc8-b48c94ed6f71"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:52.381250026Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1229,"":"io task","client":"1","":"downstairs","session_id":"14548c6d-7114-4a90-9fc8-b48c94ed6f71"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:52.381384212Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1229,"task":"crutest"} {"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:52.381398362Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1229} {"msg":"1e32d877-17d0-4e71-bf75-f10cf94290e5 active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:52.381430001Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1229,"session_id":"14548c6d-7114-4a90-9fc8-b48c94ed6f71"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:52.381439181Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1229,"client":"0","":"downstairs","session_id":"14548c6d-7114-4a90-9fc8-b48c94ed6f71"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:52.38145066Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1229,"client":"1","":"downstairs","session_id":"14548c6d-7114-4a90-9fc8-b48c94ed6f71"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:52.38146053Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1229,"client":"2","":"downstairs","session_id":"14548c6d-7114-4a90-9fc8-b48c94ed6f71"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:52.38147018Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1229,"":"io task","client":"2","":"downstairs","session_id":"14548c6d-7114-4a90-9fc8-b48c94ed6f71"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:52.381479999Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1229,"":"io task","client":"0","":"downstairs","session_id":"14548c6d-7114-4a90-9fc8-b48c94ed6f71"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:52.381489919Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1229,"":"io task","client":"1","":"downstairs","session_id":"14548c6d-7114-4a90-9fc8-b48c94ed6f71"} {"msg":"ds_connection connected from Ok(127.0.0.1:60957)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:52.381614206Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1229,"":"io task","client":"1","":"downstairs","session_id":"14548c6d-7114-4a90-9fc8-b48c94ed6f71"} {"msg":"ds_connection connected from Ok(127.0.0.1:32950)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:52.381648485Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1229,"":"io task","client":"2","":"downstairs","session_id":"14548c6d-7114-4a90-9fc8-b48c94ed6f71"} {"msg":"ds_connection connected from Ok(127.0.0.1:35878)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:52.381674174Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1229,"":"io task","client":"0","":"downstairs","session_id":"14548c6d-7114-4a90-9fc8-b48c94ed6f71"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:52.382238338Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1229,"client":"2","":"downstairs","session_id":"14548c6d-7114-4a90-9fc8-b48c94ed6f71"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:52.382302226Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1229,"client":"0","":"downstairs","session_id":"14548c6d-7114-4a90-9fc8-b48c94ed6f71"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:52.382348024Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1229,"client":"1","":"downstairs","session_id":"14548c6d-7114-4a90-9fc8-b48c94ed6f71"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:52.382579838Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1229,"session_id":"14548c6d-7114-4a90-9fc8-b48c94ed6f71"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:52.382613017Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1229,"session_id":"14548c6d-7114-4a90-9fc8-b48c94ed6f71"} {"msg":"[0]R flush_numbers[0..12]: [26, 33, 23, 44, 44, 32, 33, 35, 40, 12, 43, 43]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:52.382696474Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1229,"":"downstairs","":"downstairs","session_id":"14548c6d-7114-4a90-9fc8-b48c94ed6f71"} {"msg":"[0]R generation[0..12]: [15, 17, 14, 23, 23, 16, 17, 18, 22, 7, 22, 22]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:52.382707534Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1229,"":"downstairs","":"downstairs","session_id":"14548c6d-7114-4a90-9fc8-b48c94ed6f71"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:52.382720474Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1229,"":"downstairs","":"downstairs","session_id":"14548c6d-7114-4a90-9fc8-b48c94ed6f71"} {"msg":"[1]R flush_numbers[0..12]: [26, 33, 23, 44, 44, 32, 33, 35, 40, 12, 43, 43]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:52.382730743Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1229,"":"downstairs","":"downstairs","session_id":"14548c6d-7114-4a90-9fc8-b48c94ed6f71"} {"msg":"[1]R generation[0..12]: [15, 17, 14, 23, 23, 16, 17, 18, 22, 7, 22, 22]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:52.382741103Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1229,"":"downstairs","":"downstairs","session_id":"14548c6d-7114-4a90-9fc8-b48c94ed6f71"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:52.382751873Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1229,"":"downstairs","":"downstairs","session_id":"14548c6d-7114-4a90-9fc8-b48c94ed6f71"} {"msg":"[2]R flush_numbers[0..12]: [26, 33, 23, 44, 44, 32, 33, 35, 40, 12, 43, 43]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:52.382762052Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1229,"":"downstairs","":"downstairs","session_id":"14548c6d-7114-4a90-9fc8-b48c94ed6f71"} {"msg":"[2]R generation[0..12]: [15, 17, 14, 23, 23, 16, 17, 18, 22, 7, 22, 22]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:52.382772262Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1229,"":"downstairs","":"downstairs","session_id":"14548c6d-7114-4a90-9fc8-b48c94ed6f71"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:52.382783112Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1229,"":"downstairs","":"downstairs","session_id":"14548c6d-7114-4a90-9fc8-b48c94ed6f71"} {"msg":"Max found gen is 24","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:52.382792472Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1229,"":"downstairs","":"downstairs","session_id":"14548c6d-7114-4a90-9fc8-b48c94ed6f71"} {"msg":"Generation requested: 24 >= found:24","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:52.382802071Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1229,"":"downstairs","":"downstairs","session_id":"14548c6d-7114-4a90-9fc8-b48c94ed6f71"} {"msg":"Next flush: 45","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:52.382811331Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1229,"":"downstairs","":"downstairs","session_id":"14548c6d-7114-4a90-9fc8-b48c94ed6f71"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:52.382820571Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1229,"":"downstairs","":"downstairs","session_id":"14548c6d-7114-4a90-9fc8-b48c94ed6f71"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:52.38283018Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1229,"session_id":"14548c6d-7114-4a90-9fc8-b48c94ed6f71"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:52.38283917Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1229,"client":"0","":"downstairs","session_id":"14548c6d-7114-4a90-9fc8-b48c94ed6f71"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:52.38284871Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1229,"client":"1","":"downstairs","session_id":"14548c6d-7114-4a90-9fc8-b48c94ed6f71"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:52.3828602Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1229,"client":"2","":"downstairs","session_id":"14548c6d-7114-4a90-9fc8-b48c94ed6f71"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:52.382870269Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1229,"session_id":"14548c6d-7114-4a90-9fc8-b48c94ed6f71"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:52.382879529Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1229,"session_id":"14548c6d-7114-4a90-9fc8-b48c94ed6f71"} {"msg":"1e32d877-17d0-4e71-bf75-f10cf94290e5 is now active with session: 14548c6d-7114-4a90-9fc8-b48c94ed6f71","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:52.382889439Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1229,"session_id":"14548c6d-7114-4a90-9fc8-b48c94ed6f71"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:52.382898028Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1229,"session_id":"14548c6d-7114-4a90-9fc8-b48c94ed6f71"} {"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:52.382907058Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1229} {"msg":"Activated sub_volume 1e32d877-17d0-4e71-bf75-f10cf94290e5","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:52.382914908Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1229} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:52.382931768Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1229,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:52.383285987Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1229,"task":"crutest"} 01/30 Flush 02/30 Flush 03/30 Write block 132 len 20480 data: 3 4 4 4 4 04/30 Write block 535 len 4096 data: 7 05/30 Read block 322 len 36864 06/30 Write block 316 len 8192 data: 5 3 07/30 Read block 41 len 28672 08/30 Write block 104 len 4096 data: 2 09/30 Read block 197 len 24576 10/30 Read block 27 len 32768 11/30 Read block 437 len 24576 12/30 Write block 261 len 8192 data: 2 2 13/30 Read block 211 len 8192 14/30 Read block 194 len 28672 15/30 Write block 381 len 24576 data: 5 5 4 4 3 3 16/30 Write block 573 len 24576 data: 3 3 3 5 5 5 17/30 Read block 533 len 20480 18/30 Write block 83 len 20480 data: 3 3 2 2 2 19/30 Flush 20/30 Write block 100 len 4096 data: 2 21/30 Write block 210 len 36864 data: 5 4 5 5 4 4 6 7 4 22/30 Read block 550 len 40960 23/30 Write block 430 len 16384 data: 2 2 3 3 24/30 Read block 395 len 4096 25/30 Read block 543 len 8192 26/30 Write block 162 len 8192 data: 3 4 27/30 Read block 365 len 16384 28/30 Write block 367 len 28672 data: 5 4 4 3 2 2 2 29/30 Read block 145 len 8192 30/30 Read block 31 len 36864 ---------------------------------------------------------------- Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Flush 0 Sent Done Done false 1001 Acked Flush 0 Sent Done Done false 1002 Acked Write 5 Sent Done Done false 1003 Acked Write 1 Sent Done Done false 1004 Acked Read 9 Sent Done Done false 1005 Acked Write 2 Sent Done Done false 1006 Acked Read 7 Sent Done Done false 1007 Acked Write 1 Sent Done Done false 1008 Acked Read 6 Sent Done Done false 1009 Acked Read 8 Sent Done Done false 1010 Acked Read 6 Sent Done Done false 1011 Acked Write 2 Sent Done Done false 1012 Acked Read 2 Sent Done Done false 1013 Acked Read 7 Sent Done Done false 1014 Acked Write 6 Sent Done Done false 1015 Acked Write 6 Sent Done Done false 1016 Acked Read 5 Sent Done Done false 1017 Acked Write 5 Sent Done Done false 1018 Acked Flush 0 Sent Done Done false 1019 Acked Write 1 Sent Done Done false 1020 Acked Write 9 Sent Done Done false 1021 Acked Read 10 Sent Done Done false 1022 Acked Write 4 Sent Done Done false 1023 Acked Read 1 Sent Done Done false 1024 Acked Read 2 Sent Done Done false 1025 Acked Write 2 Sent Done Done false 1026 Acked Read 4 Sent Done Done false 1027 Acked Write 7 Sent Done Done false 1028 Acked Read 2 Sent Done Done false 1029 Acked Read 9 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 1 0 31 Done 0 29 30 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:42:52.391784302Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1229} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:52.391959737Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1229,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 15 15 15 26 26 26 F F F 1 020-039 17 17 17 33 33 33 F F F 2 040-059 14 14 14 23 23 23 F F F 3 060-079 23 23 23 44 44 44 F F F 4 080-099 23 24 24 44 47 47 F F F <--- 5 100-119 16 24 24 32 47 47 F T T <--- 6 120-139 17 24 24 33 47 47 F F F <--- 7 140-159 18 18 18 35 35 35 F F F 8 160-179 22 22 22 40 40 40 F T T <--- 9 180-199 7 7 7 12 12 12 F F F 10 200-219 22 22 22 43 43 43 F T T <--- 11 220-239 22 22 22 43 43 43 F F F 12 240-259 22 22 22 43 43 43 F F F 13 260-279 18 24 24 35 47 47 F F F <--- 14 280-299 14 14 14 24 24 24 F F F 15 300-319 22 24 24 40 47 47 F F F <--- 16 320-339 8 8 8 15 15 15 F F F 17 340-359 20 20 20 38 38 38 F F F 18 360-379 19 19 19 37 37 37 F T T <--- 19 380-399 18 24 24 36 47 47 F F F <--- 20 400-419 15 15 15 26 26 26 F F F 21 420-439 22 22 22 41 41 41 F T T <--- 22 440-459 22 22 22 41 41 41 F F F 23 460-479 18 18 18 36 36 36 F F F 24 480-499 21 21 21 39 39 39 F F F 25 500-519 21 21 21 39 39 39 F F F 26 520-539 22 24 24 43 47 47 F F F <--- 27 540-559 22 22 22 41 41 41 F F F 28 560-579 15 24 24 26 47 47 F F F <--- 29 580-599 21 21 21 39 39 39 F F F Max gen: 24, Max flush: 47 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 12 Verifying data now No change after verify Loop: 12 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 15 15 15 26 26 26 F F F 1 020-039 17 17 17 33 33 33 F F F 2 040-059 14 14 14 23 23 23 F F F 3 060-079 23 23 23 44 44 44 F F F 4 080-099 24 24 24 47 47 47 F F F 5 100-119 25 25 25 48 48 48 F F F 6 120-139 24 24 24 47 47 47 F F F 7 140-159 18 18 18 35 35 35 F F F 8 160-179 25 25 25 48 48 48 F F F 9 180-199 7 7 7 12 12 12 F F F 10 200-219 25 25 25 48 48 48 F F F 11 220-239 22 22 22 43 43 43 F F F 12 240-259 22 22 22 43 43 43 F F F 13 260-279 24 24 24 47 47 47 F F F 14 280-299 14 14 14 24 24 24 F F F 15 300-319 24 24 24 47 47 47 F F F 16 320-339 8 8 8 15 15 15 F F F 17 340-359 20 20 20 38 38 38 F F F 18 360-379 25 25 25 48 48 48 F F F 19 380-399 24 24 24 47 47 47 F F F 20 400-419 15 15 15 26 26 26 F F F 21 420-439 25 25 25 48 48 48 F F F 22 440-459 22 22 22 41 41 41 F F F 23 460-479 18 18 18 36 36 36 F F F 24 480-499 21 21 21 39 39 39 F F F 25 500-519 21 21 21 39 39 39 F F F 26 520-539 24 24 24 47 47 47 F F F 27 540-559 22 22 22 41 41 41 F F F 28 560-579 24 24 24 47 47 47 F F F 29 580-599 21 21 21 39 39 39 F F F Max gen: 25, Max flush: 48 Begin loop 13 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:56.640849119Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1237,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:56.648953976Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1237,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:56.649243847Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1237} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:56.649269597Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1237} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:56.649278136Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1237} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:56.649289516Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1237} {"msg":"Crucible d093947b-961f-4009-9bb5-6d8a60cbf80b has session id: cbd1a30a-c47e-49a0-9de8-da0ba6e62923","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:56.649693304Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1237,"session_id":"cbd1a30a-c47e-49a0-9de8-da0ba6e62923"} {"msg":"Upstairs opts: Upstairs UUID: d093947b-961f-4009-9bb5-6d8a60cbf80b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:56.649723554Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1237,"session_id":"cbd1a30a-c47e-49a0-9de8-da0ba6e62923"} {"msg":"Crucible stats registered with UUID: d093947b-961f-4009-9bb5-6d8a60cbf80b","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:56.649737223Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1237,"session_id":"cbd1a30a-c47e-49a0-9de8-da0ba6e62923"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:42:56.649746483Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1237,"session_id":"cbd1a30a-c47e-49a0-9de8-da0ba6e62923"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:56.649882919Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1237,"":"io task","client":"0","":"downstairs","session_id":"cbd1a30a-c47e-49a0-9de8-da0ba6e62923"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:56.649915638Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1237,"":"io task","client":"1","":"downstairs","session_id":"cbd1a30a-c47e-49a0-9de8-da0ba6e62923"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:56.649926498Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1237,"":"io task","client":"2","":"downstairs","session_id":"cbd1a30a-c47e-49a0-9de8-da0ba6e62923"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:56.650080853Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1237,"task":"crutest"} {"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:56.650089983Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1237} {"msg":"d093947b-961f-4009-9bb5-6d8a60cbf80b active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:56.650106522Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1237,"session_id":"cbd1a30a-c47e-49a0-9de8-da0ba6e62923"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:56.650115382Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1237,"client":"0","":"downstairs","session_id":"cbd1a30a-c47e-49a0-9de8-da0ba6e62923"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:56.650125072Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1237,"client":"1","":"downstairs","session_id":"cbd1a30a-c47e-49a0-9de8-da0ba6e62923"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:56.650134632Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1237,"client":"2","":"downstairs","session_id":"cbd1a30a-c47e-49a0-9de8-da0ba6e62923"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:56.650144041Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1237,"":"io task","client":"2","":"downstairs","session_id":"cbd1a30a-c47e-49a0-9de8-da0ba6e62923"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:56.650153941Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1237,"":"io task","client":"0","":"downstairs","session_id":"cbd1a30a-c47e-49a0-9de8-da0ba6e62923"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:56.650163821Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1237,"":"io task","client":"1","":"downstairs","session_id":"cbd1a30a-c47e-49a0-9de8-da0ba6e62923"} {"msg":"ds_connection connected from Ok(127.0.0.1:56236)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:56.650267938Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1237,"":"io task","client":"0","":"downstairs","session_id":"cbd1a30a-c47e-49a0-9de8-da0ba6e62923"} {"msg":"ds_connection connected from Ok(127.0.0.1:65427)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:56.650295687Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1237,"":"io task","client":"2","":"downstairs","session_id":"cbd1a30a-c47e-49a0-9de8-da0ba6e62923"} {"msg":"ds_connection connected from Ok(127.0.0.1:61446)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:56.650306917Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1237,"":"io task","client":"1","":"downstairs","session_id":"cbd1a30a-c47e-49a0-9de8-da0ba6e62923"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:56.651075954Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1237,"client":"2","":"downstairs","session_id":"cbd1a30a-c47e-49a0-9de8-da0ba6e62923"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:56.651132073Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1237,"client":"1","":"downstairs","session_id":"cbd1a30a-c47e-49a0-9de8-da0ba6e62923"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:56.651180011Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1237,"client":"0","":"downstairs","session_id":"cbd1a30a-c47e-49a0-9de8-da0ba6e62923"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:56.651386586Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1237,"session_id":"cbd1a30a-c47e-49a0-9de8-da0ba6e62923"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:56.651405715Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1237,"session_id":"cbd1a30a-c47e-49a0-9de8-da0ba6e62923"} {"msg":"[0]R flush_numbers[0..12]: [26, 33, 23, 44, 47, 48, 47, 35, 48, 12, 48, 43]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:56.651514382Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1237,"":"downstairs","":"downstairs","session_id":"cbd1a30a-c47e-49a0-9de8-da0ba6e62923"} {"msg":"[0]R generation[0..12]: [15, 17, 14, 23, 24, 25, 24, 18, 25, 7, 25, 22]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:56.651525502Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1237,"":"downstairs","":"downstairs","session_id":"cbd1a30a-c47e-49a0-9de8-da0ba6e62923"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:56.651536451Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1237,"":"downstairs","":"downstairs","session_id":"cbd1a30a-c47e-49a0-9de8-da0ba6e62923"} {"msg":"[1]R flush_numbers[0..12]: [26, 33, 23, 44, 47, 48, 47, 35, 48, 12, 48, 43]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:56.651546861Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1237,"":"downstairs","":"downstairs","session_id":"cbd1a30a-c47e-49a0-9de8-da0ba6e62923"} {"msg":"[1]R generation[0..12]: [15, 17, 14, 23, 24, 25, 24, 18, 25, 7, 25, 22]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:56.651557201Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1237,"":"downstairs","":"downstairs","session_id":"cbd1a30a-c47e-49a0-9de8-da0ba6e62923"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:56.65156805Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1237,"":"downstairs","":"downstairs","session_id":"cbd1a30a-c47e-49a0-9de8-da0ba6e62923"} {"msg":"[2]R flush_numbers[0..12]: [26, 33, 23, 44, 47, 48, 47, 35, 48, 12, 48, 43]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:56.65157871Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1237,"":"downstairs","":"downstairs","session_id":"cbd1a30a-c47e-49a0-9de8-da0ba6e62923"} {"msg":"[2]R generation[0..12]: [15, 17, 14, 23, 24, 25, 24, 18, 25, 7, 25, 22]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:56.65159121Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1237,"":"downstairs","":"downstairs","session_id":"cbd1a30a-c47e-49a0-9de8-da0ba6e62923"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:56.651602049Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1237,"":"downstairs","":"downstairs","session_id":"cbd1a30a-c47e-49a0-9de8-da0ba6e62923"} {"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:56.651611449Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1237,"":"downstairs","":"downstairs","session_id":"cbd1a30a-c47e-49a0-9de8-da0ba6e62923"} {"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:56.651621009Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1237,"":"downstairs","":"downstairs","session_id":"cbd1a30a-c47e-49a0-9de8-da0ba6e62923"} {"msg":"Next flush: 49","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:56.651630199Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1237,"":"downstairs","":"downstairs","session_id":"cbd1a30a-c47e-49a0-9de8-da0ba6e62923"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:56.651639478Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1237,"":"downstairs","":"downstairs","session_id":"cbd1a30a-c47e-49a0-9de8-da0ba6e62923"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:56.651649218Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1237,"session_id":"cbd1a30a-c47e-49a0-9de8-da0ba6e62923"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:56.651657988Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1237,"client":"0","":"downstairs","session_id":"cbd1a30a-c47e-49a0-9de8-da0ba6e62923"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:56.651667647Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1237,"client":"1","":"downstairs","session_id":"cbd1a30a-c47e-49a0-9de8-da0ba6e62923"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:56.651677337Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1237,"client":"2","":"downstairs","session_id":"cbd1a30a-c47e-49a0-9de8-da0ba6e62923"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:56.651686997Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1237,"session_id":"cbd1a30a-c47e-49a0-9de8-da0ba6e62923"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:56.651695987Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1237,"session_id":"cbd1a30a-c47e-49a0-9de8-da0ba6e62923"} {"msg":"d093947b-961f-4009-9bb5-6d8a60cbf80b is now active with session: cbd1a30a-c47e-49a0-9de8-da0ba6e62923","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:56.651705976Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1237,"session_id":"cbd1a30a-c47e-49a0-9de8-da0ba6e62923"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:56.651714556Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1237,"session_id":"cbd1a30a-c47e-49a0-9de8-da0ba6e62923"} {"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:56.651723626Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1237} {"msg":"Activated sub_volume d093947b-961f-4009-9bb5-6d8a60cbf80b","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:56.651731586Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1237} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:56.651739495Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1237,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:56.652039627Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1237,"task":"crutest"} 01/30 Write block 548 len 12288 data: 4 4 5 02/30 Read block 85 len 28672 03/30 Read block 205 len 16384 04/30 Write block 29 len 28672 data: 3 3 4 4 4 3 3 05/30 Read block 274 len 16384 06/30 Write block 17 len 36864 data: 3 3 3 3 2 3 2 2 2 07/30 Read block 108 len 28672 08/30 Read block 518 len 36864 09/30 Write block 271 len 12288 data: 3 3 4 10/30 Read block 321 len 40960 11/30 Write block 360 len 28672 data: 2 3 3 4 5 5 5 12/30 Read block 274 len 24576 13/30 Read block 100 len 8192 14/30 Write block 278 len 8192 data: 2 2 15/30 Flush 16/30 Write block 299 len 32768 data: 3 3 3 2 2 3 3 3 17/30 Read block 107 len 16384 18/30 Read block 379 len 4096 19/30 Flush 20/30 Write block 239 len 12288 data: 3 3 3 21/30 Write block 486 len 24576 data: 4 5 4 2 2 3 22/30 Write block 160 len 4096 data: 2 23/30 Write block 156 len 4096 data: 2 24/30 Read block 491 len 20480 25/30 Read block 334 len 12288 26/30 Write block 505 len 28672 data: 3 3 4 4 4 5 6 27/30 Write block 423 len 32768 data: 3 3 2 2 2 2 2 3 28/30 Read block 559 len 8192 29/30 Read block 376 len 36864 30/30 Read block 346 len 36864 ---------------------------------------------------------------- Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Read 7 Done Done Done false 1002 Acked Read 4 Done Done Done false 1003 Acked Write 7 Done Sent Done false 1004 Acked Read 4 Done Sent Done false 1005 Acked Write 9 Done Sent Done false 1006 Acked Read 7 Done Sent Done false 1007 Acked Read 9 Done Sent Done false 1008 Acked Write 3 Done Sent Done false 1009 Acked Read 10 Done Sent Done false 1010 Acked Write 7 Done Sent Done false 1011 Acked Read 6 Done Sent Done false 1012 Acked Read 2 Done Sent Done false 1013 Acked Write 2 Done Sent Done false 1014 Acked Flush 0 Done Sent Done false 1015 Acked Write 8 Done Sent Done false 1016 Acked Read 4 Done Sent Done false 1017 Acked Read 1 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Write 3 Done Sent Done false 1020 Acked Write 6 Done Sent Done false 1021 Acked Write 1 Done Sent Done false 1022 Acked Write 1 Done Sent Done false 1023 Acked Read 5 Done Sent Done false 1024 Acked Read 3 Done Sent Done false 1025 Acked Write 7 Done Sent Done false 1026 Acked Write 8 Done Sent Done false 1027 Acked Read 2 Done Sent Done false 1028 Acked Read 9 Done Sent Done false 1029 Acked Read 9 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 27 0 27 Done 30 3 30 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) None Some(JobId(1018)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:42:56.661195373Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1237} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:42:56.661370218Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1237,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 15 26 49 26 49 F F F <--- 1 020-039 26 17 26 49 33 49 F F F <--- 2 040-059 14 14 14 23 23 23 F F F 3 060-079 23 23 23 44 44 44 F F F 4 080-099 24 24 24 47 47 47 F F F 5 100-119 25 25 25 48 48 48 F F F 6 120-139 24 24 24 47 47 47 F F F 7 140-159 18 18 18 35 35 35 T F T <--- 8 160-179 25 25 25 48 48 48 T F T <--- 9 180-199 7 7 7 12 12 12 F F F 10 200-219 25 25 25 48 48 48 F F F 11 220-239 22 22 22 43 43 43 T F T <--- 12 240-259 22 22 22 43 43 43 T F T <--- 13 260-279 26 24 26 49 47 49 F F F <--- 14 280-299 26 14 26 50 24 50 F F F <--- 15 300-319 26 24 26 50 47 50 F F F <--- 16 320-339 8 8 8 15 15 15 F F F 17 340-359 20 20 20 38 38 38 F F F 18 360-379 26 25 26 49 48 49 F F F <--- 19 380-399 24 24 24 47 47 47 F F F 20 400-419 15 15 15 26 26 26 F F F 21 420-439 25 25 25 48 48 48 T F T <--- 22 440-459 22 22 22 41 41 41 F F F 23 460-479 18 18 18 36 36 36 F F F 24 480-499 21 21 21 39 39 39 T F T <--- 25 500-519 21 21 21 39 39 39 T F T <--- 26 520-539 24 24 24 47 47 47 F F F 27 540-559 26 22 26 49 41 49 F T F <--- 28 560-579 24 24 24 47 47 47 F F F 29 580-599 21 21 21 39 39 39 F F F Max gen: 26, Max flush: 50 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 13 Verifying data now No change after verify Loop: 13 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 49 49 49 F F F 1 020-039 26 26 26 49 49 49 F F F 2 040-059 14 14 14 23 23 23 F F F 3 060-079 23 23 23 44 44 44 F F F 4 080-099 24 24 24 47 47 47 F F F 5 100-119 25 25 25 48 48 48 F F F 6 120-139 24 24 24 47 47 47 F F F 7 140-159 27 27 27 51 51 51 F F F 8 160-179 27 27 27 51 51 51 F F F 9 180-199 7 7 7 12 12 12 F F F 10 200-219 25 25 25 48 48 48 F F F 11 220-239 27 27 27 51 51 51 F F F 12 240-259 27 27 27 51 51 51 F F F 13 260-279 26 26 26 49 49 49 F F F 14 280-299 26 26 26 50 50 50 F F F 15 300-319 26 26 26 50 50 50 F F F 16 320-339 8 8 8 15 15 15 F F F 17 340-359 20 20 20 38 38 38 F F F 18 360-379 26 26 26 49 49 49 F F F 19 380-399 24 24 24 47 47 47 F F F 20 400-419 15 15 15 26 26 26 F F F 21 420-439 27 27 27 51 51 51 F F F 22 440-459 22 22 22 41 41 41 F F F 23 460-479 18 18 18 36 36 36 F F F 24 480-499 27 27 27 51 51 51 F F F 25 500-519 27 27 27 51 51 51 F F F 26 520-539 24 24 24 47 47 47 F F F 27 540-559 26 26 26 49 49 49 F F F 28 560-579 24 24 24 47 47 47 F F F 29 580-599 21 21 21 39 39 39 F F F Max gen: 27, Max flush: 51 Begin loop 14 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:00.911649509Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1245,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:43:00.918785544Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1245,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:00.918850762Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1245,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:00.925914558Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1245,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:00.92620212Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1245} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:00.926228059Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1245} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:00.926236669Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1245} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:00.926248198Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1245} {"msg":"Crucible bd873011-5a89-4dd8-b493-7709a785a774 has session id: c20a0eb5-df2d-4cc9-9c6b-a8de12427ae7","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:00.926592328Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1245,"session_id":"c20a0eb5-df2d-4cc9-9c6b-a8de12427ae7"} {"msg":"Upstairs opts: Upstairs UUID: bd873011-5a89-4dd8-b493-7709a785a774, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:00.926615358Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1245,"session_id":"c20a0eb5-df2d-4cc9-9c6b-a8de12427ae7"} {"msg":"Crucible stats registered with UUID: bd873011-5a89-4dd8-b493-7709a785a774","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:00.926625337Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1245,"session_id":"c20a0eb5-df2d-4cc9-9c6b-a8de12427ae7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:43:00.926634347Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1245,"session_id":"c20a0eb5-df2d-4cc9-9c6b-a8de12427ae7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:00.926790123Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1245,"":"io task","client":"0","":"downstairs","session_id":"c20a0eb5-df2d-4cc9-9c6b-a8de12427ae7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:00.926822042Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1245,"":"io task","client":"2","":"downstairs","session_id":"c20a0eb5-df2d-4cc9-9c6b-a8de12427ae7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:00.926834941Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1245,"":"io task","client":"1","":"downstairs","session_id":"c20a0eb5-df2d-4cc9-9c6b-a8de12427ae7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:00.926988507Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1245,"task":"crutest"} {"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:00.926999397Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1245} {"msg":"bd873011-5a89-4dd8-b493-7709a785a774 active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:00.927035796Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1245,"session_id":"c20a0eb5-df2d-4cc9-9c6b-a8de12427ae7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:00.927045265Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1245,"client":"0","":"downstairs","session_id":"c20a0eb5-df2d-4cc9-9c6b-a8de12427ae7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:00.927055045Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1245,"client":"1","":"downstairs","session_id":"c20a0eb5-df2d-4cc9-9c6b-a8de12427ae7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:00.927064805Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1245,"client":"2","":"downstairs","session_id":"c20a0eb5-df2d-4cc9-9c6b-a8de12427ae7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:00.927074164Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1245,"":"io task","client":"2","":"downstairs","session_id":"c20a0eb5-df2d-4cc9-9c6b-a8de12427ae7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:00.927084084Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1245,"":"io task","client":"0","":"downstairs","session_id":"c20a0eb5-df2d-4cc9-9c6b-a8de12427ae7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:00.927093844Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1245,"":"io task","client":"1","":"downstairs","session_id":"c20a0eb5-df2d-4cc9-9c6b-a8de12427ae7"} {"msg":"ds_connection connected from Ok(127.0.0.1:47693)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:00.927154012Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1245,"":"io task","client":"0","":"downstairs","session_id":"c20a0eb5-df2d-4cc9-9c6b-a8de12427ae7"} {"msg":"ds_connection connected from Ok(127.0.0.1:63648)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:00.927180101Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1245,"":"io task","client":"2","":"downstairs","session_id":"c20a0eb5-df2d-4cc9-9c6b-a8de12427ae7"} {"msg":"ds_connection connected from Ok(127.0.0.1:44060)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:00.927276389Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1245,"":"io task","client":"1","":"downstairs","session_id":"c20a0eb5-df2d-4cc9-9c6b-a8de12427ae7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:00.927849302Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1245,"client":"1","":"downstairs","session_id":"c20a0eb5-df2d-4cc9-9c6b-a8de12427ae7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:00.92793679Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1245,"client":"2","":"downstairs","session_id":"c20a0eb5-df2d-4cc9-9c6b-a8de12427ae7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:00.927970929Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1245,"client":"0","":"downstairs","session_id":"c20a0eb5-df2d-4cc9-9c6b-a8de12427ae7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:00.928148323Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1245,"session_id":"c20a0eb5-df2d-4cc9-9c6b-a8de12427ae7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:00.928168053Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1245,"session_id":"c20a0eb5-df2d-4cc9-9c6b-a8de12427ae7"} {"msg":"[0]R flush_numbers[0..12]: [49, 49, 23, 44, 47, 48, 47, 51, 51, 12, 48, 51]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:00.92825882Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1245,"":"downstairs","":"downstairs","session_id":"c20a0eb5-df2d-4cc9-9c6b-a8de12427ae7"} {"msg":"[0]R generation[0..12]: [26, 26, 14, 23, 24, 25, 24, 27, 27, 7, 25, 27]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:00.9282699Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1245,"":"downstairs","":"downstairs","session_id":"c20a0eb5-df2d-4cc9-9c6b-a8de12427ae7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:00.92828122Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1245,"":"downstairs","":"downstairs","session_id":"c20a0eb5-df2d-4cc9-9c6b-a8de12427ae7"} {"msg":"[1]R flush_numbers[0..12]: [49, 49, 23, 44, 47, 48, 47, 51, 51, 12, 48, 51]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:00.928291559Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1245,"":"downstairs","":"downstairs","session_id":"c20a0eb5-df2d-4cc9-9c6b-a8de12427ae7"} {"msg":"[1]R generation[0..12]: [26, 26, 14, 23, 24, 25, 24, 27, 27, 7, 25, 27]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:00.928301899Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1245,"":"downstairs","":"downstairs","session_id":"c20a0eb5-df2d-4cc9-9c6b-a8de12427ae7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:00.928312689Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1245,"":"downstairs","":"downstairs","session_id":"c20a0eb5-df2d-4cc9-9c6b-a8de12427ae7"} {"msg":"[2]R flush_numbers[0..12]: [49, 49, 23, 44, 47, 48, 47, 51, 51, 12, 48, 51]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:00.928322868Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1245,"":"downstairs","":"downstairs","session_id":"c20a0eb5-df2d-4cc9-9c6b-a8de12427ae7"} {"msg":"[2]R generation[0..12]: [26, 26, 14, 23, 24, 25, 24, 27, 27, 7, 25, 27]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:00.928333188Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1245,"":"downstairs","":"downstairs","session_id":"c20a0eb5-df2d-4cc9-9c6b-a8de12427ae7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:00.928343968Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1245,"":"downstairs","":"downstairs","session_id":"c20a0eb5-df2d-4cc9-9c6b-a8de12427ae7"} {"msg":"Max found gen is 28","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:00.928353388Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1245,"":"downstairs","":"downstairs","session_id":"c20a0eb5-df2d-4cc9-9c6b-a8de12427ae7"} {"msg":"Generation requested: 28 >= found:28","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:00.928363087Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1245,"":"downstairs","":"downstairs","session_id":"c20a0eb5-df2d-4cc9-9c6b-a8de12427ae7"} {"msg":"Next flush: 52","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:00.928372417Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1245,"":"downstairs","":"downstairs","session_id":"c20a0eb5-df2d-4cc9-9c6b-a8de12427ae7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:00.928383237Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1245,"":"downstairs","":"downstairs","session_id":"c20a0eb5-df2d-4cc9-9c6b-a8de12427ae7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:00.928392876Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1245,"session_id":"c20a0eb5-df2d-4cc9-9c6b-a8de12427ae7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:00.928401626Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1245,"client":"0","":"downstairs","session_id":"c20a0eb5-df2d-4cc9-9c6b-a8de12427ae7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:00.928411286Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1245,"client":"1","":"downstairs","session_id":"c20a0eb5-df2d-4cc9-9c6b-a8de12427ae7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:00.928421076Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1245,"client":"2","":"downstairs","session_id":"c20a0eb5-df2d-4cc9-9c6b-a8de12427ae7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:00.928430885Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1245,"session_id":"c20a0eb5-df2d-4cc9-9c6b-a8de12427ae7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:00.928439975Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1245,"session_id":"c20a0eb5-df2d-4cc9-9c6b-a8de12427ae7"} {"msg":"bd873011-5a89-4dd8-b493-7709a785a774 is now active with session: c20a0eb5-df2d-4cc9-9c6b-a8de12427ae7","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:00.928449905Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1245,"session_id":"c20a0eb5-df2d-4cc9-9c6b-a8de12427ae7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:00.928458664Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1245,"session_id":"c20a0eb5-df2d-4cc9-9c6b-a8de12427ae7"} {"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:00.928467604Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1245} {"msg":"Activated sub_volume bd873011-5a89-4dd8-b493-7709a785a774","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:00.928475934Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1245} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:00.928484124Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1245,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:00.928800375Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1245,"task":"crutest"} 01/30 Write block 145 len 32768 data: 4 5 4 4 3 3 2 2 02/30 Write block 511 len 36864 data: 7 4 3 2 3 3 2 2 3 03/30 Write block 470 len 4096 data: 2 04/30 Read block 563 len 24576 05/30 Read block 461 len 12288 06/30 Read block 217 len 32768 07/30 Read block 60 len 36864 08/30 Flush 09/30 Write block 188 len 12288 data: 2 2 2 10/30 Read block 326 len 40960 11/30 Write block 522 len 12288 data: 5 5 5 12/30 Write block 519 len 4096 data: 4 13/30 Read block 93 len 16384 14/30 Read block 430 len 32768 15/30 Read block 398 len 4096 16/30 Write block 407 len 28672 data: 4 3 3 3 4 4 4 17/30 Read block 264 len 40960 18/30 Read block 133 len 32768 19/30 Write block 2 len 4096 data: 3 20/30 Write block 456 len 12288 data: 2 2 2 21/30 Flush 22/30 Write block 77 len 40960 data: 5 5 4 4 3 3 4 4 3 3 23/30 Write block 52 len 16384 data: 2 2 3 3 24/30 Read block 264 len 8192 25/30 Write block 22 len 8192 data: 4 3 26/30 Read block 427 len 12288 27/30 Write block 48 len 4096 data: 2 28/30 Read block 271 len 16384 29/30 Write block 168 len 16384 data: 5 6 6 6 30/30 Write block 564 len 12288 data: 2 2 2 ---------------------------------------------------------------- Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Write 9 Done Done Done false 1002 Acked Write 1 Sent Done Done false 1003 Acked Read 6 Sent Done Done false 1004 Acked Read 3 Sent Done Done false 1005 Acked Read 8 Sent Done Done false 1006 Acked Read 9 Sent Done Done false 1007 Acked Flush 0 Sent Done Done false 1008 Acked Write 3 Sent Done Done false 1009 Acked Read 10 Sent Done Done false 1010 Acked Write 3 Sent Done Done false 1011 Acked Write 1 Sent Done Done false 1012 Acked Read 4 Sent Done Done false 1013 Acked Read 8 Sent Done Done false 1014 Acked Read 1 Sent Done Done false 1015 Acked Write 7 Sent Done Done false 1016 Acked Read 10 Sent Done Done false 1017 Acked Read 8 Sent Done Done false 1018 Acked Write 1 Sent Done Done false 1019 Acked Write 3 Sent Done Done false 1020 Acked Flush 0 Sent Done Done false 1021 Acked Write 10 Sent Done Done false 1022 Acked Write 4 Sent Done Done false 1023 Acked Read 2 Sent Done Done false 1024 Acked Write 2 Sent Done Done false 1025 Acked Read 3 Sent Done Done false 1026 Acked Write 1 Sent Done Done false 1027 Acked Read 4 Sent Done Done false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 2 2 32 Done 2 28 28 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:43:00.936951609Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1245} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:00.937127004Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1245,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 28 28 49 53 53 F F F <--- 1 020-039 26 26 26 49 49 49 F T T <--- 2 040-059 14 14 14 23 23 23 F T T <--- 3 060-079 23 23 23 44 44 44 F T T <--- 4 080-099 24 24 24 47 47 47 F T T <--- 5 100-119 25 25 25 48 48 48 F F F 6 120-139 24 24 24 47 47 47 F F F 7 140-159 27 28 28 51 52 52 T F F <--- 8 160-179 27 27 27 51 51 51 F T T <--- 9 180-199 7 28 28 12 53 53 F F F <--- 10 200-219 25 25 25 48 48 48 F F F 11 220-239 27 27 27 51 51 51 F F F 12 240-259 27 27 27 51 51 51 F F F 13 260-279 26 26 26 49 49 49 F F F 14 280-299 26 26 26 50 50 50 F F F 15 300-319 26 26 26 50 50 50 F F F 16 320-339 8 8 8 15 15 15 F F F 17 340-359 20 20 20 38 38 38 F F F 18 360-379 26 26 26 49 49 49 F F F 19 380-399 24 24 24 47 47 47 F F F 20 400-419 15 28 28 26 53 53 F F F <--- 21 420-439 27 27 27 51 51 51 F F F 22 440-459 22 28 28 41 53 53 F F F <--- 23 460-479 18 28 28 36 52 52 F F F <--- 24 480-499 27 27 27 51 51 51 F F F 25 500-519 27 28 28 51 53 53 T F F <--- 26 520-539 24 28 28 47 53 53 F F F <--- 27 540-559 26 26 26 49 49 49 F F F 28 560-579 24 24 24 47 47 47 F T T <--- 29 580-599 21 21 21 39 39 39 F F F Max gen: 28, Max flush: 53 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 14 Verifying data now No change after verify Loop: 14 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 28 28 28 53 53 53 F F F 1 020-039 29 29 29 54 54 54 F F F 2 040-059 29 29 29 54 54 54 F F F 3 060-079 29 29 29 54 54 54 F F F 4 080-099 29 29 29 54 54 54 F F F 5 100-119 25 25 25 48 48 48 F F F 6 120-139 24 24 24 47 47 47 F F F 7 140-159 28 28 28 52 52 52 F F F 8 160-179 29 29 29 54 54 54 F F F 9 180-199 28 28 28 53 53 53 F F F 10 200-219 25 25 25 48 48 48 F F F 11 220-239 27 27 27 51 51 51 F F F 12 240-259 27 27 27 51 51 51 F F F 13 260-279 26 26 26 49 49 49 F F F 14 280-299 26 26 26 50 50 50 F F F 15 300-319 26 26 26 50 50 50 F F F 16 320-339 8 8 8 15 15 15 F F F 17 340-359 20 20 20 38 38 38 F F F 18 360-379 26 26 26 49 49 49 F F F 19 380-399 24 24 24 47 47 47 F F F 20 400-419 28 28 28 53 53 53 F F F 21 420-439 27 27 27 51 51 51 F F F 22 440-459 28 28 28 53 53 53 F F F 23 460-479 28 28 28 52 52 52 F F F 24 480-499 27 27 27 51 51 51 F F F 25 500-519 28 28 28 53 53 53 F F F 26 520-539 28 28 28 53 53 53 F F F 27 540-559 26 26 26 49 49 49 F F F 28 560-579 29 29 29 54 54 54 F F F 29 580-599 21 21 21 39 39 39 F F F Max gen: 29, Max flush: 54 Begin loop 15 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:05.191129279Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1253,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:05.199300954Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1253,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:05.199579216Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1253} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:05.199604045Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1253} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:05.199614645Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1253} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:05.199626184Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1253} {"msg":"Crucible bc9a0970-0e5b-470c-b5b6-72d107c7b14a has session id: a328d58c-22bc-4a67-a781-03887bce2fb1","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:05.199967824Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1253,"session_id":"a328d58c-22bc-4a67-a781-03887bce2fb1"} {"msg":"Upstairs opts: Upstairs UUID: bc9a0970-0e5b-470c-b5b6-72d107c7b14a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:05.200012593Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1253,"session_id":"a328d58c-22bc-4a67-a781-03887bce2fb1"} {"msg":"Crucible stats registered with UUID: bc9a0970-0e5b-470c-b5b6-72d107c7b14a","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:05.200038842Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1253,"session_id":"a328d58c-22bc-4a67-a781-03887bce2fb1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:43:05.200048882Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1253,"session_id":"a328d58c-22bc-4a67-a781-03887bce2fb1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:05.200179728Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1253,"":"io task","client":"0","":"downstairs","session_id":"a328d58c-22bc-4a67-a781-03887bce2fb1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:05.200212767Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1253,"":"io task","client":"1","":"downstairs","session_id":"a328d58c-22bc-4a67-a781-03887bce2fb1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:05.200223477Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1253,"":"io task","client":"2","":"downstairs","session_id":"a328d58c-22bc-4a67-a781-03887bce2fb1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:05.200379063Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1253,"task":"crutest"} {"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:05.200389702Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1253} {"msg":"bc9a0970-0e5b-470c-b5b6-72d107c7b14a active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:05.200405742Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1253,"session_id":"a328d58c-22bc-4a67-a781-03887bce2fb1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:05.200414661Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1253,"client":"0","":"downstairs","session_id":"a328d58c-22bc-4a67-a781-03887bce2fb1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:05.200424241Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1253,"client":"1","":"downstairs","session_id":"a328d58c-22bc-4a67-a781-03887bce2fb1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:05.200433941Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1253,"client":"2","":"downstairs","session_id":"a328d58c-22bc-4a67-a781-03887bce2fb1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:05.200445781Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1253,"":"io task","client":"2","":"downstairs","session_id":"a328d58c-22bc-4a67-a781-03887bce2fb1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:05.20045567Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1253,"":"io task","client":"0","":"downstairs","session_id":"a328d58c-22bc-4a67-a781-03887bce2fb1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:05.2004659Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1253,"":"io task","client":"1","":"downstairs","session_id":"a328d58c-22bc-4a67-a781-03887bce2fb1"} {"msg":"ds_connection connected from Ok(127.0.0.1:38680)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:05.200523828Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1253,"":"io task","client":"0","":"downstairs","session_id":"a328d58c-22bc-4a67-a781-03887bce2fb1"} {"msg":"ds_connection connected from Ok(127.0.0.1:58014)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:05.200534808Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1253,"":"io task","client":"2","":"downstairs","session_id":"a328d58c-22bc-4a67-a781-03887bce2fb1"} {"msg":"ds_connection connected from Ok(127.0.0.1:54507)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:05.200637345Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1253,"":"io task","client":"1","":"downstairs","session_id":"a328d58c-22bc-4a67-a781-03887bce2fb1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:05.201193409Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1253,"client":"2","":"downstairs","session_id":"a328d58c-22bc-4a67-a781-03887bce2fb1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:05.201219778Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1253,"client":"1","":"downstairs","session_id":"a328d58c-22bc-4a67-a781-03887bce2fb1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:05.201328945Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1253,"client":"0","":"downstairs","session_id":"a328d58c-22bc-4a67-a781-03887bce2fb1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:05.201566258Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1253,"session_id":"a328d58c-22bc-4a67-a781-03887bce2fb1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:05.201598437Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1253,"session_id":"a328d58c-22bc-4a67-a781-03887bce2fb1"} {"msg":"[0]R flush_numbers[0..12]: [53, 54, 54, 54, 54, 48, 47, 52, 54, 53, 48, 51]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:05.201645486Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1253,"":"downstairs","":"downstairs","session_id":"a328d58c-22bc-4a67-a781-03887bce2fb1"} {"msg":"[0]R generation[0..12]: [28, 29, 29, 29, 29, 25, 24, 28, 29, 28, 25, 27]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:05.201657326Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1253,"":"downstairs","":"downstairs","session_id":"a328d58c-22bc-4a67-a781-03887bce2fb1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:05.201668415Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1253,"":"downstairs","":"downstairs","session_id":"a328d58c-22bc-4a67-a781-03887bce2fb1"} {"msg":"[1]R flush_numbers[0..12]: [53, 54, 54, 54, 54, 48, 47, 52, 54, 53, 48, 51]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:05.201681715Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1253,"":"downstairs","":"downstairs","session_id":"a328d58c-22bc-4a67-a781-03887bce2fb1"} {"msg":"[1]R generation[0..12]: [28, 29, 29, 29, 29, 25, 24, 28, 29, 28, 25, 27]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:05.201692185Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1253,"":"downstairs","":"downstairs","session_id":"a328d58c-22bc-4a67-a781-03887bce2fb1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:05.201703124Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1253,"":"downstairs","":"downstairs","session_id":"a328d58c-22bc-4a67-a781-03887bce2fb1"} {"msg":"[2]R flush_numbers[0..12]: [53, 54, 54, 54, 54, 48, 47, 52, 54, 53, 48, 51]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:05.201713514Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1253,"":"downstairs","":"downstairs","session_id":"a328d58c-22bc-4a67-a781-03887bce2fb1"} {"msg":"[2]R generation[0..12]: [28, 29, 29, 29, 29, 25, 24, 28, 29, 28, 25, 27]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:05.201723714Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1253,"":"downstairs","":"downstairs","session_id":"a328d58c-22bc-4a67-a781-03887bce2fb1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:05.201734453Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1253,"":"downstairs","":"downstairs","session_id":"a328d58c-22bc-4a67-a781-03887bce2fb1"} {"msg":"Max found gen is 30","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:05.201743793Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1253,"":"downstairs","":"downstairs","session_id":"a328d58c-22bc-4a67-a781-03887bce2fb1"} {"msg":"Generation requested: 30 >= found:30","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:05.201753383Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1253,"":"downstairs","":"downstairs","session_id":"a328d58c-22bc-4a67-a781-03887bce2fb1"} {"msg":"Next flush: 55","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:05.201762623Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1253,"":"downstairs","":"downstairs","session_id":"a328d58c-22bc-4a67-a781-03887bce2fb1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:05.201771952Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1253,"":"downstairs","":"downstairs","session_id":"a328d58c-22bc-4a67-a781-03887bce2fb1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:05.201781822Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1253,"session_id":"a328d58c-22bc-4a67-a781-03887bce2fb1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:05.201790572Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1253,"client":"0","":"downstairs","session_id":"a328d58c-22bc-4a67-a781-03887bce2fb1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:05.201800192Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1253,"client":"1","":"downstairs","session_id":"a328d58c-22bc-4a67-a781-03887bce2fb1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:05.201809711Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1253,"client":"2","":"downstairs","session_id":"a328d58c-22bc-4a67-a781-03887bce2fb1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:05.201819361Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1253,"session_id":"a328d58c-22bc-4a67-a781-03887bce2fb1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:05.201828241Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1253,"session_id":"a328d58c-22bc-4a67-a781-03887bce2fb1"} {"msg":"bc9a0970-0e5b-470c-b5b6-72d107c7b14a is now active with session: a328d58c-22bc-4a67-a781-03887bce2fb1","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:05.20183927Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1253,"session_id":"a328d58c-22bc-4a67-a781-03887bce2fb1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:05.20184807Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1253,"session_id":"a328d58c-22bc-4a67-a781-03887bce2fb1"} {"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:05.2018574Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1253} {"msg":"Activated sub_volume bc9a0970-0e5b-470c-b5b6-72d107c7b14a","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:05.20186557Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1253} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:05.201873519Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1253,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:05.20220708Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1253,"task":"crutest"} 01/30 Write block 595 len 12288 data: 4 4 3 02/30 Flush 03/30 Write block 289 len 36864 data: 3 3 3 3 4 5 5 5 5 04/30 Write block 559 len 4096 data: 5 05/30 Write block 400 len 20480 data: 3 3 3 3 3 06/30 Read block 103 len 16384 07/30 Read block 13 len 28672 08/30 Write block 136 len 16384 data: 5 3 3 3 09/30 Write block 483 len 12288 data: 3 3 4 10/30 Write block 396 len 40960 data: 2 3 3 4 4 4 4 4 4 4 11/30 Read block 110 len 36864 12/30 Read block 323 len 4096 13/30 Read block 385 len 16384 14/30 Read block 223 len 20480 15/30 Read block 334 len 28672 16/30 Write block 376 len 40960 data: 4 4 4 6 6 6 6 5 5 4 17/30 Read block 489 len 28672 18/30 Write block 483 len 16384 data: 4 4 5 5 19/30 Flush 20/30 Write block 375 len 16384 data: 4 5 5 5 21/30 Write block 526 len 28672 data: 3 3 3 3 2 2 5 22/30 Read block 395 len 40960 23/30 Read block 159 len 16384 24/30 Write block 335 len 28672 data: 3 4 5 3 3 3 3 25/30 Read block 429 len 40960 26/30 Write block 552 len 36864 data: 3 3 2 2 3 5 5 6 3 27/30 Read block 518 len 8192 28/30 Read block 544 len 36864 29/30 Write block 330 len 32768 data: 4 4 3 3 3 4 5 6 30/30 Write block 458 len 4096 data: 3 ---------------------------------------------------------------- Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Sent Done false 1001 Acked Flush 0 Done Sent Done false 1002 Acked Write 9 Done Sent Done false 1003 Acked Write 1 Done Sent Done false 1004 Acked Write 5 Done Sent Done false 1005 Acked Read 4 Done Sent Done false 1006 Acked Read 7 Done Sent Done false 1007 Acked Write 4 Done Sent Done false 1008 Acked Write 3 Done Sent Done false 1009 Acked Write 10 Done Sent Done false 1010 Acked Read 9 Done Sent Done false 1011 Acked Read 1 Done Sent Done false 1012 Acked Read 4 Done Sent Done false 1013 Acked Read 5 Done Sent Done false 1014 Acked Read 7 Done Sent Done false 1015 Acked Write 10 Done Sent Done false 1016 Acked Read 7 Done Sent Done false 1017 Acked Write 4 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Write 4 Done Sent Done false 1020 Acked Write 7 Done Sent Done false 1021 Acked Read 10 Done Sent Done false 1022 Acked Read 4 Done Sent Done false 1023 Acked Write 7 Done Sent Done false 1024 Acked Read 10 Done Sent Done false 1025 Acked Write 9 Done Sent Done false 1026 Acked Read 2 Done Sent Done false 1027 Acked Read 9 Done Sent Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 30 2 34 Done 28 0 28 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) None Some(JobId(1018)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:43:05.210892509Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1253} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:05.211059094Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1253,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 28 28 28 53 53 53 F F F 1 020-039 29 29 29 54 54 54 F F F 2 040-059 29 29 29 54 54 54 F F F 3 060-079 29 29 29 54 54 54 F F F 4 080-099 29 29 29 54 54 54 F F F 5 100-119 25 25 25 48 48 48 F F F 6 120-139 30 24 30 56 47 56 F F F <--- 7 140-159 28 28 28 52 52 52 F F F 8 160-179 29 29 29 54 54 54 F F F 9 180-199 28 28 28 53 53 53 F F F 10 200-219 25 25 25 48 48 48 F F F 11 220-239 27 27 27 51 51 51 F F F 12 240-259 27 27 27 51 51 51 F F F 13 260-279 26 26 26 49 49 49 F F F 14 280-299 30 26 30 56 50 56 F F F <--- 15 300-319 26 26 26 50 50 50 F F F 16 320-339 8 8 8 15 15 15 T F T <--- 17 340-359 20 20 20 38 38 38 T F T <--- 18 360-379 30 26 30 56 49 56 T F T <--- 19 380-399 30 24 30 56 47 56 F F F <--- 20 400-419 30 28 30 56 53 56 F F F <--- 21 420-439 27 27 27 51 51 51 F F F 22 440-459 28 28 28 53 53 53 T F T <--- 23 460-479 28 28 28 52 52 52 F F F 24 480-499 30 27 30 56 51 56 F F F <--- 25 500-519 28 28 28 53 53 53 F F F 26 520-539 28 28 28 53 53 53 T F T <--- 27 540-559 30 26 30 56 49 56 T F T <--- 28 560-579 29 29 29 54 54 54 T F T <--- 29 580-599 30 21 30 55 39 55 F F F <--- Max gen: 30, Max flush: 56 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 15 Verifying data now No change after verify Loop: 15 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 28 28 28 53 53 53 F F F 1 020-039 29 29 29 54 54 54 F F F 2 040-059 29 29 29 54 54 54 F F F 3 060-079 29 29 29 54 54 54 F F F 4 080-099 29 29 29 54 54 54 F F F 5 100-119 25 25 25 48 48 48 F F F 6 120-139 30 30 30 56 56 56 F F F 7 140-159 28 28 28 52 52 52 F F F 8 160-179 29 29 29 54 54 54 F F F 9 180-199 28 28 28 53 53 53 F F F 10 200-219 25 25 25 48 48 48 F F F 11 220-239 27 27 27 51 51 51 F F F 12 240-259 27 27 27 51 51 51 F F F 13 260-279 26 26 26 49 49 49 F F F 14 280-299 30 30 30 56 56 56 F F F 15 300-319 26 26 26 50 50 50 F F F 16 320-339 31 31 31 57 57 57 F F F 17 340-359 31 31 31 57 57 57 F F F 18 360-379 31 31 31 57 57 57 F F F 19 380-399 30 30 30 56 56 56 F F F 20 400-419 30 30 30 56 56 56 F F F 21 420-439 27 27 27 51 51 51 F F F 22 440-459 31 31 31 57 57 57 F F F 23 460-479 28 28 28 52 52 52 F F F 24 480-499 30 30 30 56 56 56 F F F 25 500-519 28 28 28 53 53 53 F F F 26 520-539 31 31 31 57 57 57 F F F 27 540-559 31 31 31 57 57 57 F F F 28 560-579 31 31 31 57 57 57 F F F 29 580-599 30 30 30 55 55 55 F F F Max gen: 31, Max flush: 57 Begin loop 16 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:09.455709119Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1261,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:09.463714608Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1261,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:09.46401468Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1261} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:09.464038949Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1261} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:09.464047689Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1261} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:09.464059178Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1261} {"msg":"Crucible 8c404a31-df41-48ad-92fd-8c982434695d has session id: 9bd26e95-22af-4261-995b-ea3a9bf0ece1","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:09.464398688Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1261,"session_id":"9bd26e95-22af-4261-995b-ea3a9bf0ece1"} {"msg":"Upstairs opts: Upstairs UUID: 8c404a31-df41-48ad-92fd-8c982434695d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:09.464427668Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1261,"session_id":"9bd26e95-22af-4261-995b-ea3a9bf0ece1"} {"msg":"Crucible stats registered with UUID: 8c404a31-df41-48ad-92fd-8c982434695d","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:09.464440997Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1261,"session_id":"9bd26e95-22af-4261-995b-ea3a9bf0ece1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:43:09.464450367Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1261,"session_id":"9bd26e95-22af-4261-995b-ea3a9bf0ece1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:09.464566404Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1261,"":"io task","client":"0","":"downstairs","session_id":"9bd26e95-22af-4261-995b-ea3a9bf0ece1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:09.464599403Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1261,"":"io task","client":"1","":"downstairs","session_id":"9bd26e95-22af-4261-995b-ea3a9bf0ece1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:09.464610352Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1261,"":"io task","client":"2","":"downstairs","session_id":"9bd26e95-22af-4261-995b-ea3a9bf0ece1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:09.464742799Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1261,"task":"crutest"} {"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:09.464760708Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1261} {"msg":"8c404a31-df41-48ad-92fd-8c982434695d active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:09.464787687Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1261,"session_id":"9bd26e95-22af-4261-995b-ea3a9bf0ece1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:09.464796937Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1261,"client":"0","":"downstairs","session_id":"9bd26e95-22af-4261-995b-ea3a9bf0ece1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:09.464806637Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1261,"client":"1","":"downstairs","session_id":"9bd26e95-22af-4261-995b-ea3a9bf0ece1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:09.464816266Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1261,"client":"2","":"downstairs","session_id":"9bd26e95-22af-4261-995b-ea3a9bf0ece1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:09.464825736Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1261,"":"io task","client":"2","":"downstairs","session_id":"9bd26e95-22af-4261-995b-ea3a9bf0ece1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:09.464835616Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1261,"":"io task","client":"0","":"downstairs","session_id":"9bd26e95-22af-4261-995b-ea3a9bf0ece1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:09.464845426Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1261,"":"io task","client":"1","":"downstairs","session_id":"9bd26e95-22af-4261-995b-ea3a9bf0ece1"} {"msg":"ds_connection connected from Ok(127.0.0.1:40285)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:09.464939333Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1261,"":"io task","client":"2","":"downstairs","session_id":"9bd26e95-22af-4261-995b-ea3a9bf0ece1"} {"msg":"ds_connection connected from Ok(127.0.0.1:33426)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:09.464964872Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1261,"":"io task","client":"0","":"downstairs","session_id":"9bd26e95-22af-4261-995b-ea3a9bf0ece1"} {"msg":"ds_connection connected from Ok(127.0.0.1:54733)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:09.465062349Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1261,"":"io task","client":"1","":"downstairs","session_id":"9bd26e95-22af-4261-995b-ea3a9bf0ece1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:09.465593214Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1261,"client":"1","":"downstairs","session_id":"9bd26e95-22af-4261-995b-ea3a9bf0ece1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:09.465612553Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1261,"client":"2","":"downstairs","session_id":"9bd26e95-22af-4261-995b-ea3a9bf0ece1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:09.465623553Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1261,"client":"0","":"downstairs","session_id":"9bd26e95-22af-4261-995b-ea3a9bf0ece1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:09.465888885Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1261,"session_id":"9bd26e95-22af-4261-995b-ea3a9bf0ece1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:09.465898435Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1261,"session_id":"9bd26e95-22af-4261-995b-ea3a9bf0ece1"} {"msg":"[0]R flush_numbers[0..12]: [53, 54, 54, 54, 54, 48, 56, 52, 54, 53, 48, 51]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:09.465927824Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1261,"":"downstairs","":"downstairs","session_id":"9bd26e95-22af-4261-995b-ea3a9bf0ece1"} {"msg":"[0]R generation[0..12]: [28, 29, 29, 29, 29, 25, 30, 28, 29, 28, 25, 27]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:09.465938924Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1261,"":"downstairs","":"downstairs","session_id":"9bd26e95-22af-4261-995b-ea3a9bf0ece1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:09.465949994Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1261,"":"downstairs","":"downstairs","session_id":"9bd26e95-22af-4261-995b-ea3a9bf0ece1"} {"msg":"[1]R flush_numbers[0..12]: [53, 54, 54, 54, 54, 48, 56, 52, 54, 53, 48, 51]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:09.465960403Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1261,"":"downstairs","":"downstairs","session_id":"9bd26e95-22af-4261-995b-ea3a9bf0ece1"} {"msg":"[1]R generation[0..12]: [28, 29, 29, 29, 29, 25, 30, 28, 29, 28, 25, 27]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:09.465970693Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1261,"":"downstairs","":"downstairs","session_id":"9bd26e95-22af-4261-995b-ea3a9bf0ece1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:09.465981603Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1261,"":"downstairs","":"downstairs","session_id":"9bd26e95-22af-4261-995b-ea3a9bf0ece1"} {"msg":"[2]R flush_numbers[0..12]: [53, 54, 54, 54, 54, 48, 56, 52, 54, 53, 48, 51]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:09.465991903Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1261,"":"downstairs","":"downstairs","session_id":"9bd26e95-22af-4261-995b-ea3a9bf0ece1"} {"msg":"[2]R generation[0..12]: [28, 29, 29, 29, 29, 25, 30, 28, 29, 28, 25, 27]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:09.466012772Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1261,"":"downstairs","":"downstairs","session_id":"9bd26e95-22af-4261-995b-ea3a9bf0ece1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:09.466023622Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1261,"":"downstairs","":"downstairs","session_id":"9bd26e95-22af-4261-995b-ea3a9bf0ece1"} {"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:09.466035011Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1261,"":"downstairs","":"downstairs","session_id":"9bd26e95-22af-4261-995b-ea3a9bf0ece1"} {"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:09.466044781Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1261,"":"downstairs","":"downstairs","session_id":"9bd26e95-22af-4261-995b-ea3a9bf0ece1"} {"msg":"Next flush: 58","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:09.466054221Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1261,"":"downstairs","":"downstairs","session_id":"9bd26e95-22af-4261-995b-ea3a9bf0ece1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:09.46606359Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1261,"":"downstairs","":"downstairs","session_id":"9bd26e95-22af-4261-995b-ea3a9bf0ece1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:09.46607341Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1261,"session_id":"9bd26e95-22af-4261-995b-ea3a9bf0ece1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:09.46608206Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1261,"client":"0","":"downstairs","session_id":"9bd26e95-22af-4261-995b-ea3a9bf0ece1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:09.46609163Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1261,"client":"1","":"downstairs","session_id":"9bd26e95-22af-4261-995b-ea3a9bf0ece1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:09.466101279Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1261,"client":"2","":"downstairs","session_id":"9bd26e95-22af-4261-995b-ea3a9bf0ece1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:09.466115769Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1261,"session_id":"9bd26e95-22af-4261-995b-ea3a9bf0ece1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:09.466125199Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1261,"session_id":"9bd26e95-22af-4261-995b-ea3a9bf0ece1"} {"msg":"8c404a31-df41-48ad-92fd-8c982434695d is now active with session: 9bd26e95-22af-4261-995b-ea3a9bf0ece1","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:09.466135078Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1261,"session_id":"9bd26e95-22af-4261-995b-ea3a9bf0ece1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:09.466143778Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1261,"session_id":"9bd26e95-22af-4261-995b-ea3a9bf0ece1"} {"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:09.466152818Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1261} {"msg":"Activated sub_volume 8c404a31-df41-48ad-92fd-8c982434695d","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:09.466160868Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1261} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:09.466168937Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1261,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:09.46643345Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1261,"task":"crutest"} 01/30 Write block 16 len 16384 data: 3 4 4 4 02/30 Read block 349 len 40960 03/30 Read block 90 len 4096 04/30 Read block 125 len 12288 05/30 Read block 301 len 40960 06/30 Read block 32 len 12288 07/30 Read block 544 len 28672 08/30 Write block 202 len 20480 data: 5 6 6 5 5 09/30 Read block 168 len 28672 10/30 Read block 370 len 20480 11/30 Write block 466 len 28672 data: 2 2 2 2 3 2 2 12/30 Read block 226 len 28672 13/30 Read block 59 len 16384 14/30 Write block 218 len 40960 data: 5 4 3 3 4 6 6 6 6 5 15/30 Read block 456 len 24576 16/30 Read block 541 len 8192 17/30 Write block 498 len 28672 data: 4 4 4 4 4 4 4 18/30 Read block 80 len 24576 19/30 Read block 196 len 8192 20/30 Read block 340 len 24576 21/30 Flush 22/30 Write block 387 len 12288 data: 3 3 2 23/30 Read block 269 len 24576 24/30 Read block 189 len 16384 25/30 Read block 404 len 28672 26/30 Read block 68 len 20480 27/30 Read block 483 len 32768 28/30 Read block 426 len 12288 29/30 Read block 417 len 28672 30/30 Read block 560 len 24576 ---------------------------------------------------------------- Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Read 10 Done Done Done false 1002 Acked Read 1 Done Done Done false 1003 Acked Read 3 Done Done Done false 1004 Acked Read 10 Done Done Done false 1005 Acked Read 3 Done Done Sent false 1006 Acked Read 7 Done Done Sent false 1007 Acked Write 5 Done Done Sent false 1008 Acked Read 7 Done Done Sent false 1009 Acked Read 5 Done Done Sent false 1010 Acked Write 7 Done Done Sent false 1011 Acked Read 7 Done Done Sent false 1012 Acked Read 4 Done Done Sent false 1013 Acked Write 10 Done Done Sent false 1014 Acked Read 6 Done Done Sent false 1015 Acked Read 2 Done Done Sent false 1016 Acked Write 7 Done Done Sent false 1017 Acked Read 6 Done Done Sent false 1018 Acked Read 2 Done Done Sent false 1019 Acked Read 6 Done Done Sent false 1020 Acked Flush 0 Done Done Sent false 1021 Acked Write 3 Done Done Sent false 1022 Acked Read 6 Done Done Sent false 1023 Acked Read 4 Done Done Sent false 1024 Acked Read 7 Done Done Sent false 1025 Acked Read 5 Done Done Sent false 1026 Acked Read 8 Done Done Sent false 1027 Acked Read 3 Done Done Sent false 1028 Acked Read 7 Done Done Sent false 1029 Acked Read 6 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 25 26 Done 29 30 5 64 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:43:09.475822109Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1261} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:09.475999344Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1261,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 32 32 28 58 58 53 F F T <--- 1 020-039 29 29 29 54 54 54 F F F 2 040-059 29 29 29 54 54 54 F F F 3 060-079 29 29 29 54 54 54 F F F 4 080-099 29 29 29 54 54 54 F F F 5 100-119 25 25 25 48 48 48 F F F 6 120-139 30 30 30 56 56 56 F F F 7 140-159 28 28 28 52 52 52 F F F 8 160-179 29 29 29 54 54 54 F F F 9 180-199 28 28 28 53 53 53 F F F 10 200-219 32 32 25 58 58 48 F F F <--- 11 220-239 32 32 27 58 58 51 F F F <--- 12 240-259 27 27 27 51 51 51 F F F 13 260-279 26 26 26 49 49 49 F F F 14 280-299 30 30 30 56 56 56 F F F 15 300-319 26 26 26 50 50 50 F F F 16 320-339 31 31 31 57 57 57 F F F 17 340-359 31 31 31 57 57 57 F F F 18 360-379 31 31 31 57 57 57 F F F 19 380-399 30 30 30 56 56 56 T T F <--- 20 400-419 30 30 30 56 56 56 F F F 21 420-439 27 27 27 51 51 51 F F F 22 440-459 31 31 31 57 57 57 F F F 23 460-479 32 32 28 58 58 52 F F F <--- 24 480-499 32 32 30 58 58 56 F F F <--- 25 500-519 32 32 28 58 58 53 F F F <--- 26 520-539 31 31 31 57 57 57 F F F 27 540-559 31 31 31 57 57 57 F F F 28 560-579 31 31 31 57 57 57 F F F 29 580-599 30 30 30 55 55 55 F F F Max gen: 32, Max flush: 58 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 16 Verifying data now No change after verify Loop: 16 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 32 32 32 58 58 58 F F F 1 020-039 29 29 29 54 54 54 F F F 2 040-059 29 29 29 54 54 54 F F F 3 060-079 29 29 29 54 54 54 F F F 4 080-099 29 29 29 54 54 54 F F F 5 100-119 25 25 25 48 48 48 F F F 6 120-139 30 30 30 56 56 56 F F F 7 140-159 28 28 28 52 52 52 F F F 8 160-179 29 29 29 54 54 54 F F F 9 180-199 28 28 28 53 53 53 F F F 10 200-219 32 32 32 58 58 58 F F F 11 220-239 32 32 32 58 58 58 F F F 12 240-259 27 27 27 51 51 51 F F F 13 260-279 26 26 26 49 49 49 F F F 14 280-299 30 30 30 56 56 56 F F F 15 300-319 26 26 26 50 50 50 F F F 16 320-339 31 31 31 57 57 57 F F F 17 340-359 31 31 31 57 57 57 F F F 18 360-379 31 31 31 57 57 57 F F F 19 380-399 33 33 33 59 59 59 F F F 20 400-419 30 30 30 56 56 56 F F F 21 420-439 27 27 27 51 51 51 F F F 22 440-459 31 31 31 57 57 57 F F F 23 460-479 32 32 32 58 58 58 F F F 24 480-499 32 32 32 58 58 58 F F F 25 500-519 32 32 32 58 58 58 F F F 26 520-539 31 31 31 57 57 57 F F F 27 540-559 31 31 31 57 57 57 F F F 28 560-579 31 31 31 57 57 57 F F F 29 580-599 30 30 30 55 55 55 F F F Max gen: 33, Max flush: 59 Begin loop 17 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:13.705448637Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1269,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:13.713370899Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1269,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:13.71366682Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1269} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:13.713701119Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1269} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:13.713710149Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1269} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:13.713721628Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1269} {"msg":"Crucible 28aad43f-f68d-4f55-be2a-7c4b0ab3fbf5 has session id: eb9ebe1c-577f-4a69-a3d2-5d8dcefafac5","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:13.71401175Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1269,"session_id":"eb9ebe1c-577f-4a69-a3d2-5d8dcefafac5"} {"msg":"Upstairs opts: Upstairs UUID: 28aad43f-f68d-4f55-be2a-7c4b0ab3fbf5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:13.71402806Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1269,"session_id":"eb9ebe1c-577f-4a69-a3d2-5d8dcefafac5"} {"msg":"Crucible stats registered with UUID: 28aad43f-f68d-4f55-be2a-7c4b0ab3fbf5","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:13.714038089Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1269,"session_id":"eb9ebe1c-577f-4a69-a3d2-5d8dcefafac5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:43:13.714047579Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1269,"session_id":"eb9ebe1c-577f-4a69-a3d2-5d8dcefafac5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:13.714204904Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1269,"":"io task","client":"1","":"downstairs","session_id":"eb9ebe1c-577f-4a69-a3d2-5d8dcefafac5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:13.714236524Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1269,"":"io task","client":"0","":"downstairs","session_id":"eb9ebe1c-577f-4a69-a3d2-5d8dcefafac5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:13.714247413Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1269,"":"io task","client":"2","":"downstairs","session_id":"eb9ebe1c-577f-4a69-a3d2-5d8dcefafac5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:13.714395039Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1269,"task":"crutest"} {"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:13.714406679Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1269} {"msg":"28aad43f-f68d-4f55-be2a-7c4b0ab3fbf5 active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:13.714429508Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1269,"session_id":"eb9ebe1c-577f-4a69-a3d2-5d8dcefafac5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:13.714440668Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1269,"client":"0","":"downstairs","session_id":"eb9ebe1c-577f-4a69-a3d2-5d8dcefafac5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:13.714450777Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1269,"client":"1","":"downstairs","session_id":"eb9ebe1c-577f-4a69-a3d2-5d8dcefafac5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:13.714460447Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1269,"client":"2","":"downstairs","session_id":"eb9ebe1c-577f-4a69-a3d2-5d8dcefafac5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:13.714469797Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1269,"":"io task","client":"2","":"downstairs","session_id":"eb9ebe1c-577f-4a69-a3d2-5d8dcefafac5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:13.714479857Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1269,"":"io task","client":"0","":"downstairs","session_id":"eb9ebe1c-577f-4a69-a3d2-5d8dcefafac5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:13.714489776Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1269,"":"io task","client":"1","":"downstairs","session_id":"eb9ebe1c-577f-4a69-a3d2-5d8dcefafac5"} {"msg":"ds_connection connected from Ok(127.0.0.1:46382)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:13.714546035Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1269,"":"io task","client":"2","":"downstairs","session_id":"eb9ebe1c-577f-4a69-a3d2-5d8dcefafac5"} {"msg":"ds_connection connected from Ok(127.0.0.1:38777)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:13.714556804Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1269,"":"io task","client":"0","":"downstairs","session_id":"eb9ebe1c-577f-4a69-a3d2-5d8dcefafac5"} {"msg":"ds_connection connected from Ok(127.0.0.1:44648)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:13.714567124Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1269,"":"io task","client":"1","":"downstairs","session_id":"eb9ebe1c-577f-4a69-a3d2-5d8dcefafac5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:13.715216295Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1269,"client":"1","":"downstairs","session_id":"eb9ebe1c-577f-4a69-a3d2-5d8dcefafac5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:13.715237965Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1269,"client":"0","":"downstairs","session_id":"eb9ebe1c-577f-4a69-a3d2-5d8dcefafac5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:13.715280303Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1269,"client":"2","":"downstairs","session_id":"eb9ebe1c-577f-4a69-a3d2-5d8dcefafac5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:13.715516997Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1269,"session_id":"eb9ebe1c-577f-4a69-a3d2-5d8dcefafac5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:13.715541926Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1269,"session_id":"eb9ebe1c-577f-4a69-a3d2-5d8dcefafac5"} {"msg":"[0]R flush_numbers[0..12]: [58, 54, 54, 54, 54, 48, 56, 52, 54, 53, 58, 58]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:13.715600524Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1269,"":"downstairs","":"downstairs","session_id":"eb9ebe1c-577f-4a69-a3d2-5d8dcefafac5"} {"msg":"[0]R generation[0..12]: [32, 29, 29, 29, 29, 25, 30, 28, 29, 28, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:13.715613914Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1269,"":"downstairs","":"downstairs","session_id":"eb9ebe1c-577f-4a69-a3d2-5d8dcefafac5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:13.715624843Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1269,"":"downstairs","":"downstairs","session_id":"eb9ebe1c-577f-4a69-a3d2-5d8dcefafac5"} {"msg":"[1]R flush_numbers[0..12]: [58, 54, 54, 54, 54, 48, 56, 52, 54, 53, 58, 58]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:13.715635133Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1269,"":"downstairs","":"downstairs","session_id":"eb9ebe1c-577f-4a69-a3d2-5d8dcefafac5"} {"msg":"[1]R generation[0..12]: [32, 29, 29, 29, 29, 25, 30, 28, 29, 28, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:13.715645373Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1269,"":"downstairs","":"downstairs","session_id":"eb9ebe1c-577f-4a69-a3d2-5d8dcefafac5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:13.715656253Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1269,"":"downstairs","":"downstairs","session_id":"eb9ebe1c-577f-4a69-a3d2-5d8dcefafac5"} {"msg":"[2]R flush_numbers[0..12]: [58, 54, 54, 54, 54, 48, 56, 52, 54, 53, 58, 58]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:13.715666522Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1269,"":"downstairs","":"downstairs","session_id":"eb9ebe1c-577f-4a69-a3d2-5d8dcefafac5"} {"msg":"[2]R generation[0..12]: [32, 29, 29, 29, 29, 25, 30, 28, 29, 28, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:13.715676642Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1269,"":"downstairs","":"downstairs","session_id":"eb9ebe1c-577f-4a69-a3d2-5d8dcefafac5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:13.715687422Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1269,"":"downstairs","":"downstairs","session_id":"eb9ebe1c-577f-4a69-a3d2-5d8dcefafac5"} {"msg":"Max found gen is 34","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:13.715696831Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1269,"":"downstairs","":"downstairs","session_id":"eb9ebe1c-577f-4a69-a3d2-5d8dcefafac5"} {"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:13.715706551Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1269,"":"downstairs","":"downstairs","session_id":"eb9ebe1c-577f-4a69-a3d2-5d8dcefafac5"} {"msg":"Next flush: 60","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:13.715715831Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1269,"":"downstairs","":"downstairs","session_id":"eb9ebe1c-577f-4a69-a3d2-5d8dcefafac5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:13.715725081Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1269,"":"downstairs","":"downstairs","session_id":"eb9ebe1c-577f-4a69-a3d2-5d8dcefafac5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:13.71573506Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1269,"session_id":"eb9ebe1c-577f-4a69-a3d2-5d8dcefafac5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:13.71574408Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1269,"client":"0","":"downstairs","session_id":"eb9ebe1c-577f-4a69-a3d2-5d8dcefafac5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:13.7157536Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1269,"client":"1","":"downstairs","session_id":"eb9ebe1c-577f-4a69-a3d2-5d8dcefafac5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:13.715764549Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1269,"client":"2","":"downstairs","session_id":"eb9ebe1c-577f-4a69-a3d2-5d8dcefafac5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:13.715774549Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1269,"session_id":"eb9ebe1c-577f-4a69-a3d2-5d8dcefafac5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:13.715783579Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1269,"session_id":"eb9ebe1c-577f-4a69-a3d2-5d8dcefafac5"} {"msg":"28aad43f-f68d-4f55-be2a-7c4b0ab3fbf5 is now active with session: eb9ebe1c-577f-4a69-a3d2-5d8dcefafac5","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:13.715793329Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1269,"session_id":"eb9ebe1c-577f-4a69-a3d2-5d8dcefafac5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:13.715801928Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1269,"session_id":"eb9ebe1c-577f-4a69-a3d2-5d8dcefafac5"} {"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:13.715810848Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1269} {"msg":"Activated sub_volume 28aad43f-f68d-4f55-be2a-7c4b0ab3fbf5","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:13.715818768Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1269} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:13.715826738Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1269,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:13.716116149Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1269,"task":"crutest"} 01/30 Write block 13 len 24576 data: 3 3 3 4 5 5 02/30 Read block 11 len 36864 03/30 Read block 473 len 40960 04/30 Read block 582 len 32768 05/30 Write block 544 len 28672 data: 3 4 3 3 5 5 6 06/30 Read block 453 len 36864 07/30 Read block 528 len 24576 08/30 Read block 252 len 16384 09/30 Read block 103 len 8192 10/30 Flush 11/30 Write block 131 len 36864 data: 3 4 5 5 5 6 4 4 4 12/30 Read block 5 len 40960 13/30 Write block 95 len 24576 data: 3 3 2 2 2 3 14/30 Write block 359 len 28672 data: 2 3 4 4 5 6 6 15/30 Write block 364 len 40960 data: 7 7 6 6 5 5 4 3 3 3 16/30 Read block 533 len 24576 17/30 Write block 160 len 8192 data: 3 3 18/30 Write block 5 len 8192 data: 5 7 19/30 Read block 334 len 32768 20/30 Read block 544 len 40960 21/30 Read block 577 len 20480 22/30 Write block 324 len 16384 data: 3 3 3 3 23/30 Write block 93 len 32768 data: 2 3 4 4 3 3 3 4 24/30 Read block 273 len 20480 25/30 Write block 233 len 12288 data: 2 2 3 26/30 Read block 462 len 12288 27/30 Read block 498 len 24576 28/30 Write block 268 len 40960 data: 2 2 3 4 4 5 4 3 3 3 29/30 Read block 551 len 16384 30/30 Read block 18 len 32768 ---------------------------------------------------------------- Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Read 9 Done Done Done false 1002 Acked Read 10 Done Done Done false 1003 Acked Read 8 Done Done Sent false 1004 Acked Write 7 Done Done Sent false 1005 Acked Read 9 Done Done Sent false 1006 Acked Read 6 Done Done Sent false 1007 Acked Read 4 Done Done Sent false 1008 Acked Read 2 Done Done Sent false 1009 Acked Flush 0 Done Done Sent false 1010 Acked Write 9 Done Done Sent false 1011 Acked Read 10 Done Done Sent false 1012 Acked Write 6 Done Done Sent false 1013 Acked Write 7 Done Done Sent false 1014 Acked Write 10 Done Done Sent false 1015 Acked Read 6 Done Done Sent false 1016 Acked Write 2 Done Done Sent false 1017 Acked Write 2 Done Done Sent false 1018 Acked Read 8 Done Done Sent false 1019 Acked Read 10 Done Done Sent false 1020 Acked Read 5 Done Done Sent false 1021 Acked Write 4 Done Done Sent false 1022 Acked Write 8 Done Done Sent false 1023 Acked Read 5 Done Done Sent false 1024 Acked Write 3 Done Done Sent false 1025 Acked Read 3 Done Done Sent false 1026 Acked Read 6 Done Done Sent false 1027 Acked Write 10 Done Done Sent false 1028 Acked Read 4 Done Done Sent false 1029 Acked Read 8 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 27 28 Done 29 30 3 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1009)) Some(JobId(1009)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:43:13.724953724Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1269} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:13.725135889Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1269,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 32 60 60 58 T T T <--- 1 020-039 29 29 29 54 54 54 F F F 2 040-059 29 29 29 54 54 54 F F F 3 060-079 29 29 29 54 54 54 F F F 4 080-099 29 29 29 54 54 54 T T F <--- 5 100-119 25 25 25 48 48 48 T T F <--- 6 120-139 30 30 30 56 56 56 T T F <--- 7 140-159 28 28 28 52 52 52 F F F 8 160-179 29 29 29 54 54 54 T T F <--- 9 180-199 28 28 28 53 53 53 F F F 10 200-219 32 32 32 58 58 58 F F F 11 220-239 32 32 32 58 58 58 T T F <--- 12 240-259 27 27 27 51 51 51 F F F 13 260-279 26 26 26 49 49 49 T T F <--- 14 280-299 30 30 30 56 56 56 F F F 15 300-319 26 26 26 50 50 50 F F F 16 320-339 31 31 31 57 57 57 T T F <--- 17 340-359 31 31 31 57 57 57 T T F <--- 18 360-379 31 31 31 57 57 57 T T F <--- 19 380-399 33 33 33 59 59 59 F F F 20 400-419 30 30 30 56 56 56 F F F 21 420-439 27 27 27 51 51 51 F F F 22 440-459 31 31 31 57 57 57 F F F 23 460-479 32 32 32 58 58 58 F F F 24 480-499 32 32 32 58 58 58 F F F 25 500-519 32 32 32 58 58 58 F F F 26 520-539 31 31 31 57 57 57 F F F 27 540-559 34 34 31 60 60 57 F F F <--- 28 560-579 31 31 31 57 57 57 F F F 29 580-599 30 30 30 55 55 55 F F F Max gen: 34, Max flush: 60 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 17 Verifying data now No change after verify Loop: 17 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 35 35 35 61 61 61 F F F 1 020-039 29 29 29 54 54 54 F F F 2 040-059 29 29 29 54 54 54 F F F 3 060-079 29 29 29 54 54 54 F F F 4 080-099 35 35 35 61 61 61 F F F 5 100-119 35 35 35 61 61 61 F F F 6 120-139 35 35 35 61 61 61 F F F 7 140-159 28 28 28 52 52 52 F F F 8 160-179 35 35 35 61 61 61 F F F 9 180-199 28 28 28 53 53 53 F F F 10 200-219 32 32 32 58 58 58 F F F 11 220-239 35 35 35 61 61 61 F F F 12 240-259 27 27 27 51 51 51 F F F 13 260-279 35 35 35 61 61 61 F F F 14 280-299 30 30 30 56 56 56 F F F 15 300-319 26 26 26 50 50 50 F F F 16 320-339 35 35 35 61 61 61 F F F 17 340-359 35 35 35 61 61 61 F F F 18 360-379 35 35 35 61 61 61 F F F 19 380-399 33 33 33 59 59 59 F F F 20 400-419 30 30 30 56 56 56 F F F 21 420-439 27 27 27 51 51 51 F F F 22 440-459 31 31 31 57 57 57 F F F 23 460-479 32 32 32 58 58 58 F F F 24 480-499 32 32 32 58 58 58 F F F 25 500-519 32 32 32 58 58 58 F F F 26 520-539 31 31 31 57 57 57 F F F 27 540-559 34 34 34 60 60 60 F F F 28 560-579 31 31 31 57 57 57 F F F 29 580-599 30 30 30 55 55 55 F F F Max gen: 35, Max flush: 61 Begin loop 18 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:17.967289715Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1277,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:43:17.974433459Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1277,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:17.974497377Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1277,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:17.981513515Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1277,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:17.981799147Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1277} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:17.981836746Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1277} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:17.981845745Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1277} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:17.981857535Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1277} {"msg":"Crucible fde79a73-c5b6-4c94-8474-893b03c009f4 has session id: c016925a-2095-41cd-b987-3c4e0fff6f8f","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:17.982228624Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1277,"session_id":"c016925a-2095-41cd-b987-3c4e0fff6f8f"} {"msg":"Upstairs opts: Upstairs UUID: fde79a73-c5b6-4c94-8474-893b03c009f4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:17.982258514Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1277,"session_id":"c016925a-2095-41cd-b987-3c4e0fff6f8f"} {"msg":"Crucible stats registered with UUID: fde79a73-c5b6-4c94-8474-893b03c009f4","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:17.982269793Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1277,"session_id":"c016925a-2095-41cd-b987-3c4e0fff6f8f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:43:17.982279073Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1277,"session_id":"c016925a-2095-41cd-b987-3c4e0fff6f8f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:17.982415669Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1277,"":"io task","client":"0","":"downstairs","session_id":"c016925a-2095-41cd-b987-3c4e0fff6f8f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:17.982433798Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1277,"":"io task","client":"1","":"downstairs","session_id":"c016925a-2095-41cd-b987-3c4e0fff6f8f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:17.982443858Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1277,"":"io task","client":"2","":"downstairs","session_id":"c016925a-2095-41cd-b987-3c4e0fff6f8f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:17.982637463Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1277,"task":"crutest"} {"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:17.982654222Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1277} {"msg":"fde79a73-c5b6-4c94-8474-893b03c009f4 active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:17.982670152Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1277,"session_id":"c016925a-2095-41cd-b987-3c4e0fff6f8f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:17.982678981Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1277,"client":"0","":"downstairs","session_id":"c016925a-2095-41cd-b987-3c4e0fff6f8f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:17.982688591Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1277,"client":"1","":"downstairs","session_id":"c016925a-2095-41cd-b987-3c4e0fff6f8f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:17.982698291Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1277,"client":"2","":"downstairs","session_id":"c016925a-2095-41cd-b987-3c4e0fff6f8f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:17.982707911Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1277,"":"io task","client":"2","":"downstairs","session_id":"c016925a-2095-41cd-b987-3c4e0fff6f8f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:17.98271786Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1277,"":"io task","client":"0","":"downstairs","session_id":"c016925a-2095-41cd-b987-3c4e0fff6f8f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:17.982766389Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1277,"":"io task","client":"1","":"downstairs","session_id":"c016925a-2095-41cd-b987-3c4e0fff6f8f"} {"msg":"ds_connection connected from Ok(127.0.0.1:55643)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:17.982784588Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1277,"":"io task","client":"0","":"downstairs","session_id":"c016925a-2095-41cd-b987-3c4e0fff6f8f"} {"msg":"ds_connection connected from Ok(127.0.0.1:34910)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:17.982795178Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1277,"":"io task","client":"2","":"downstairs","session_id":"c016925a-2095-41cd-b987-3c4e0fff6f8f"} {"msg":"ds_connection connected from Ok(127.0.0.1:55819)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:17.982805548Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1277,"":"io task","client":"1","":"downstairs","session_id":"c016925a-2095-41cd-b987-3c4e0fff6f8f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:17.983446989Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1277,"client":"1","":"downstairs","session_id":"c016925a-2095-41cd-b987-3c4e0fff6f8f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:17.983476238Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1277,"client":"0","":"downstairs","session_id":"c016925a-2095-41cd-b987-3c4e0fff6f8f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:17.983634074Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1277,"client":"2","":"downstairs","session_id":"c016925a-2095-41cd-b987-3c4e0fff6f8f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:17.983726191Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1277,"session_id":"c016925a-2095-41cd-b987-3c4e0fff6f8f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:17.983745791Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1277,"session_id":"c016925a-2095-41cd-b987-3c4e0fff6f8f"} {"msg":"[0]R flush_numbers[0..12]: [61, 54, 54, 54, 61, 61, 61, 52, 61, 53, 58, 61]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:17.983959194Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1277,"":"downstairs","":"downstairs","session_id":"c016925a-2095-41cd-b987-3c4e0fff6f8f"} {"msg":"[0]R generation[0..12]: [35, 29, 29, 29, 35, 35, 35, 28, 35, 28, 32, 35]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:17.983970224Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1277,"":"downstairs","":"downstairs","session_id":"c016925a-2095-41cd-b987-3c4e0fff6f8f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:17.983981134Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1277,"":"downstairs","":"downstairs","session_id":"c016925a-2095-41cd-b987-3c4e0fff6f8f"} {"msg":"[1]R flush_numbers[0..12]: [61, 54, 54, 54, 61, 61, 61, 52, 61, 53, 58, 61]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:17.983991424Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1277,"":"downstairs","":"downstairs","session_id":"c016925a-2095-41cd-b987-3c4e0fff6f8f"} {"msg":"[1]R generation[0..12]: [35, 29, 29, 29, 35, 35, 35, 28, 35, 28, 32, 35]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:17.984012003Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1277,"":"downstairs","":"downstairs","session_id":"c016925a-2095-41cd-b987-3c4e0fff6f8f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:17.984026053Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1277,"":"downstairs","":"downstairs","session_id":"c016925a-2095-41cd-b987-3c4e0fff6f8f"} {"msg":"[2]R flush_numbers[0..12]: [61, 54, 54, 54, 61, 61, 61, 52, 61, 53, 58, 61]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:17.984036422Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1277,"":"downstairs","":"downstairs","session_id":"c016925a-2095-41cd-b987-3c4e0fff6f8f"} {"msg":"[2]R generation[0..12]: [35, 29, 29, 29, 35, 35, 35, 28, 35, 28, 32, 35]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:17.984046682Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1277,"":"downstairs","":"downstairs","session_id":"c016925a-2095-41cd-b987-3c4e0fff6f8f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:17.984057562Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1277,"":"downstairs","":"downstairs","session_id":"c016925a-2095-41cd-b987-3c4e0fff6f8f"} {"msg":"Max found gen is 36","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:17.984066951Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1277,"":"downstairs","":"downstairs","session_id":"c016925a-2095-41cd-b987-3c4e0fff6f8f"} {"msg":"Generation requested: 36 >= found:36","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:17.984076551Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1277,"":"downstairs","":"downstairs","session_id":"c016925a-2095-41cd-b987-3c4e0fff6f8f"} {"msg":"Next flush: 62","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:17.984085741Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1277,"":"downstairs","":"downstairs","session_id":"c016925a-2095-41cd-b987-3c4e0fff6f8f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:17.984095031Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1277,"":"downstairs","":"downstairs","session_id":"c016925a-2095-41cd-b987-3c4e0fff6f8f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:17.98411138Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1277,"session_id":"c016925a-2095-41cd-b987-3c4e0fff6f8f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:17.98412039Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1277,"client":"0","":"downstairs","session_id":"c016925a-2095-41cd-b987-3c4e0fff6f8f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:17.98413Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1277,"client":"1","":"downstairs","session_id":"c016925a-2095-41cd-b987-3c4e0fff6f8f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:17.984139699Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1277,"client":"2","":"downstairs","session_id":"c016925a-2095-41cd-b987-3c4e0fff6f8f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:17.984149849Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1277,"session_id":"c016925a-2095-41cd-b987-3c4e0fff6f8f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:17.984159079Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1277,"session_id":"c016925a-2095-41cd-b987-3c4e0fff6f8f"} {"msg":"fde79a73-c5b6-4c94-8474-893b03c009f4 is now active with session: c016925a-2095-41cd-b987-3c4e0fff6f8f","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:17.984168948Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1277,"session_id":"c016925a-2095-41cd-b987-3c4e0fff6f8f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:17.984177628Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1277,"session_id":"c016925a-2095-41cd-b987-3c4e0fff6f8f"} {"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:17.984187938Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1277} {"msg":"Activated sub_volume fde79a73-c5b6-4c94-8474-893b03c009f4","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:17.984195828Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1277} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:17.984203737Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1277,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:17.984509659Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1277,"task":"crutest"} 01/30 Write block 518 len 24576 data: 3 5 5 6 6 6 02/30 Write block 410 len 32768 data: 4 5 5 5 3 2 2 2 03/30 Read block 180 len 12288 04/30 Flush 05/30 Write block 22 len 4096 data: 5 06/30 Read block 355 len 16384 07/30 Write block 296 len 24576 data: 6 6 5 4 4 4 08/30 Read block 445 len 36864 09/30 Write block 435 len 4096 data: 4 10/30 Read block 244 len 4096 11/30 Write block 201 len 36864 data: 5 6 7 7 6 6 4 4 4 12/30 Write block 261 len 16384 data: 3 3 2 2 13/30 Write block 553 len 4096 data: 4 14/30 Read block 79 len 8192 15/30 Flush 16/30 Write block 544 len 40960 data: 4 5 4 4 6 6 7 3 4 5 17/30 Read block 413 len 40960 18/30 Read block 333 len 8192 19/30 Read block 78 len 16384 20/30 Write block 231 len 16384 data: 3 2 3 3 21/30 Write block 403 len 32768 data: 5 5 5 4 5 4 4 5 22/30 Flush 23/30 Flush 24/30 Write block 140 len 32768 data: 3 3 3 4 4 5 6 5 25/30 Write block 344 len 16384 data: 3 3 3 3 26/30 Read block 253 len 28672 27/30 Read block 184 len 28672 28/30 Read block 90 len 28672 29/30 Write block 334 len 32768 data: 4 5 6 7 4 4 4 4 30/30 Read block 380 len 40960 ---------------------------------------------------------------- Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Sent Done Done false 1001 Acked Write 8 Sent Done Done false 1002 Acked Read 3 Sent Done Done false 1003 Acked Flush 0 Sent Done Done false 1004 Acked Write 1 Sent Done Done false 1005 Acked Read 4 Sent Done Done false 1006 Acked Write 6 Sent Done Done false 1007 Acked Read 9 Sent Done Done false 1008 Acked Write 1 Sent Done Done false 1009 Acked Read 1 Sent Done Done false 1010 Acked Write 9 Sent Done Done false 1011 Acked Write 4 Sent Done Done false 1012 Acked Write 1 Sent Done Done false 1013 Acked Read 2 Sent Done Done false 1014 Acked Flush 0 Sent Done Done false 1015 Acked Write 10 Sent Done Done false 1016 Acked Read 10 Sent Done Done false 1017 Acked Read 2 Sent Done Done false 1018 Acked Read 4 Sent Done Done false 1019 Acked Write 4 Sent Done Done false 1020 Acked Write 8 Sent Done Done false 1021 Acked Flush 0 Sent Done Done false 1022 Acked Flush 0 Sent Done Done false 1023 Acked Write 8 Sent Done Done false 1024 Acked Write 4 Sent Done Done false 1025 Acked Read 7 Sent Done Done false 1026 Acked Read 7 Sent Done Done false 1027 Acked Read 7 Sent Done Done false 1028 Acked Write 8 Sent Done Done false 1029 Acked Read 10 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 0 0 30 Done 0 30 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:43:17.994334285Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1277} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:17.99450433Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1277,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 35 35 35 61 61 61 F F F 1 020-039 29 36 36 54 63 63 F F F <--- 2 040-059 29 29 29 54 54 54 F F F 3 060-079 29 29 29 54 54 54 F F F 4 080-099 35 35 35 61 61 61 F F F 5 100-119 35 35 35 61 61 61 F F F 6 120-139 35 35 35 61 61 61 F F F 7 140-159 28 28 28 52 52 52 F T T <--- 8 160-179 35 35 35 61 61 61 F F F 9 180-199 28 28 28 53 53 53 F F F 10 200-219 32 36 36 58 63 63 F F F <--- 11 220-239 35 36 36 61 64 64 F F F <--- 12 240-259 27 27 27 51 51 51 F F F 13 260-279 35 36 36 61 63 63 F F F <--- 14 280-299 30 36 36 56 63 63 F F F <--- 15 300-319 26 36 36 50 63 63 F F F <--- 16 320-339 35 35 35 61 61 61 F T T <--- 17 340-359 35 35 35 61 61 61 F T T <--- 18 360-379 35 35 35 61 61 61 F F F 19 380-399 33 33 33 59 59 59 F F F 20 400-419 30 36 36 56 64 64 F F F <--- 21 420-439 27 36 36 51 63 63 F F F <--- 22 440-459 31 31 31 57 57 57 F F F 23 460-479 32 32 32 58 58 58 F F F 24 480-499 32 32 32 58 58 58 F F F 25 500-519 32 36 36 58 62 62 F F F <--- 26 520-539 31 36 36 57 62 62 F F F <--- 27 540-559 34 36 36 60 64 64 F F F <--- 28 560-579 31 31 31 57 57 57 F F F 29 580-599 30 30 30 55 55 55 F F F Max gen: 36, Max flush: 64 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 18 Verifying data now No change after verify Loop: 18 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 35 35 35 61 61 61 F F F 1 020-039 36 36 36 63 63 63 F F F 2 040-059 29 29 29 54 54 54 F F F 3 060-079 29 29 29 54 54 54 F F F 4 080-099 35 35 35 61 61 61 F F F 5 100-119 35 35 35 61 61 61 F F F 6 120-139 35 35 35 61 61 61 F F F 7 140-159 37 37 37 65 65 65 F F F 8 160-179 35 35 35 61 61 61 F F F 9 180-199 28 28 28 53 53 53 F F F 10 200-219 36 36 36 63 63 63 F F F 11 220-239 36 36 36 64 64 64 F F F 12 240-259 27 27 27 51 51 51 F F F 13 260-279 36 36 36 63 63 63 F F F 14 280-299 36 36 36 63 63 63 F F F 15 300-319 36 36 36 63 63 63 F F F 16 320-339 37 37 37 65 65 65 F F F 17 340-359 37 37 37 65 65 65 F F F 18 360-379 35 35 35 61 61 61 F F F 19 380-399 33 33 33 59 59 59 F F F 20 400-419 36 36 36 64 64 64 F F F 21 420-439 36 36 36 63 63 63 F F F 22 440-459 31 31 31 57 57 57 F F F 23 460-479 32 32 32 58 58 58 F F F 24 480-499 32 32 32 58 58 58 F F F 25 500-519 36 36 36 62 62 62 F F F 26 520-539 36 36 36 62 62 62 F F F 27 540-559 36 36 36 64 64 64 F F F 28 560-579 31 31 31 57 57 57 F F F 29 580-599 30 30 30 55 55 55 F F F Max gen: 37, Max flush: 65 Begin loop 19 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:22.246523932Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1285,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:43:22.253645677Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1285,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:22.253714915Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1285,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:22.260614326Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1285,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:22.260895908Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1285} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:22.260923997Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1285} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:22.260932667Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1285} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:22.260944206Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1285} {"msg":"Crucible 5a3b2613-c363-4ac8-aaf8-cbc675c20761 has session id: d972663a-d2e9-42a9-a360-925f945bf523","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:22.261305286Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1285,"session_id":"d972663a-d2e9-42a9-a360-925f945bf523"} {"msg":"Upstairs opts: Upstairs UUID: 5a3b2613-c363-4ac8-aaf8-cbc675c20761, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:22.261325375Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1285,"session_id":"d972663a-d2e9-42a9-a360-925f945bf523"} {"msg":"Crucible stats registered with UUID: 5a3b2613-c363-4ac8-aaf8-cbc675c20761","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:22.261335185Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1285,"session_id":"d972663a-d2e9-42a9-a360-925f945bf523"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:43:22.261346205Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1285,"session_id":"d972663a-d2e9-42a9-a360-925f945bf523"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:22.26150137Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1285,"":"io task","client":"2","":"downstairs","session_id":"d972663a-d2e9-42a9-a360-925f945bf523"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:22.2615187Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1285,"":"io task","client":"0","":"downstairs","session_id":"d972663a-d2e9-42a9-a360-925f945bf523"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:22.26152915Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1285,"":"io task","client":"1","":"downstairs","session_id":"d972663a-d2e9-42a9-a360-925f945bf523"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:22.261685975Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1285,"task":"crutest"} {"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:22.261702655Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1285} {"msg":"5a3b2613-c363-4ac8-aaf8-cbc675c20761 active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:22.261726054Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1285,"session_id":"d972663a-d2e9-42a9-a360-925f945bf523"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:22.261735374Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1285,"client":"0","":"downstairs","session_id":"d972663a-d2e9-42a9-a360-925f945bf523"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:22.261745013Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1285,"client":"1","":"downstairs","session_id":"d972663a-d2e9-42a9-a360-925f945bf523"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:22.261754483Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1285,"client":"2","":"downstairs","session_id":"d972663a-d2e9-42a9-a360-925f945bf523"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:22.261764053Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1285,"":"io task","client":"2","":"downstairs","session_id":"d972663a-d2e9-42a9-a360-925f945bf523"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:22.261774012Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1285,"":"io task","client":"0","":"downstairs","session_id":"d972663a-d2e9-42a9-a360-925f945bf523"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:22.261783872Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1285,"":"io task","client":"1","":"downstairs","session_id":"d972663a-d2e9-42a9-a360-925f945bf523"} {"msg":"ds_connection connected from Ok(127.0.0.1:54406)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:22.261835501Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1285,"":"io task","client":"1","":"downstairs","session_id":"d972663a-d2e9-42a9-a360-925f945bf523"} {"msg":"ds_connection connected from Ok(127.0.0.1:45233)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:22.2618463Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1285,"":"io task","client":"2","":"downstairs","session_id":"d972663a-d2e9-42a9-a360-925f945bf523"} {"msg":"ds_connection connected from Ok(127.0.0.1:55853)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:22.26186109Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1285,"":"io task","client":"0","":"downstairs","session_id":"d972663a-d2e9-42a9-a360-925f945bf523"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:22.262609358Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1285,"client":"1","":"downstairs","session_id":"d972663a-d2e9-42a9-a360-925f945bf523"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:22.262634658Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1285,"client":"2","":"downstairs","session_id":"d972663a-d2e9-42a9-a360-925f945bf523"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:22.262646147Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1285,"client":"0","":"downstairs","session_id":"d972663a-d2e9-42a9-a360-925f945bf523"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:22.262928369Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1285,"session_id":"d972663a-d2e9-42a9-a360-925f945bf523"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:22.262938009Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1285,"session_id":"d972663a-d2e9-42a9-a360-925f945bf523"} {"msg":"[0]R flush_numbers[0..12]: [61, 63, 54, 54, 61, 61, 61, 65, 61, 53, 63, 64]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:22.262973498Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1285,"":"downstairs","":"downstairs","session_id":"d972663a-d2e9-42a9-a360-925f945bf523"} {"msg":"[0]R generation[0..12]: [35, 36, 29, 29, 35, 35, 35, 37, 35, 28, 36, 36]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:22.262984468Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1285,"":"downstairs","":"downstairs","session_id":"d972663a-d2e9-42a9-a360-925f945bf523"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:22.262995557Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1285,"":"downstairs","":"downstairs","session_id":"d972663a-d2e9-42a9-a360-925f945bf523"} {"msg":"[1]R flush_numbers[0..12]: [61, 63, 54, 54, 61, 61, 61, 65, 61, 53, 63, 64]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:22.263006097Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1285,"":"downstairs","":"downstairs","session_id":"d972663a-d2e9-42a9-a360-925f945bf523"} {"msg":"[1]R generation[0..12]: [35, 36, 29, 29, 35, 35, 35, 37, 35, 28, 36, 36]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:22.263016667Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1285,"":"downstairs","":"downstairs","session_id":"d972663a-d2e9-42a9-a360-925f945bf523"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:22.263027646Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1285,"":"downstairs","":"downstairs","session_id":"d972663a-d2e9-42a9-a360-925f945bf523"} {"msg":"[2]R flush_numbers[0..12]: [61, 63, 54, 54, 61, 61, 61, 65, 61, 53, 63, 64]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:22.263037896Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1285,"":"downstairs","":"downstairs","session_id":"d972663a-d2e9-42a9-a360-925f945bf523"} {"msg":"[2]R generation[0..12]: [35, 36, 29, 29, 35, 35, 35, 37, 35, 28, 36, 36]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:22.263048356Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1285,"":"downstairs","":"downstairs","session_id":"d972663a-d2e9-42a9-a360-925f945bf523"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:22.263059265Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1285,"":"downstairs","":"downstairs","session_id":"d972663a-d2e9-42a9-a360-925f945bf523"} {"msg":"Max found gen is 38","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:22.263070635Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1285,"":"downstairs","":"downstairs","session_id":"d972663a-d2e9-42a9-a360-925f945bf523"} {"msg":"Generation requested: 38 >= found:38","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:22.263080325Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1285,"":"downstairs","":"downstairs","session_id":"d972663a-d2e9-42a9-a360-925f945bf523"} {"msg":"Next flush: 66","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:22.263089575Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1285,"":"downstairs","":"downstairs","session_id":"d972663a-d2e9-42a9-a360-925f945bf523"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:22.263098924Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1285,"":"downstairs","":"downstairs","session_id":"d972663a-d2e9-42a9-a360-925f945bf523"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:22.263108474Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1285,"session_id":"d972663a-d2e9-42a9-a360-925f945bf523"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:22.263117004Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1285,"client":"0","":"downstairs","session_id":"d972663a-d2e9-42a9-a360-925f945bf523"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:22.263126853Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1285,"client":"1","":"downstairs","session_id":"d972663a-d2e9-42a9-a360-925f945bf523"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:22.263136493Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1285,"client":"2","":"downstairs","session_id":"d972663a-d2e9-42a9-a360-925f945bf523"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:22.263146213Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1285,"session_id":"d972663a-d2e9-42a9-a360-925f945bf523"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:22.263155163Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1285,"session_id":"d972663a-d2e9-42a9-a360-925f945bf523"} {"msg":"5a3b2613-c363-4ac8-aaf8-cbc675c20761 is now active with session: d972663a-d2e9-42a9-a360-925f945bf523","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:22.263165132Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1285,"session_id":"d972663a-d2e9-42a9-a360-925f945bf523"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:22.263173912Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1285,"session_id":"d972663a-d2e9-42a9-a360-925f945bf523"} {"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:22.263182982Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1285} {"msg":"Activated sub_volume 5a3b2613-c363-4ac8-aaf8-cbc675c20761","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:22.263191192Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1285} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:22.263199261Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1285,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:22.263497543Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1285,"task":"crutest"} 01/30 Flush 02/30 Flush 03/30 Write block 500 len 16384 data: 5 5 5 5 04/30 Read block 360 len 16384 05/30 Read block 582 len 24576 06/30 Read block 359 len 32768 07/30 Write block 116 len 8192 data: 4 4 08/30 Write block 362 len 16384 data: 5 6 8 8 09/30 Write block 323 len 12288 data: 3 4 4 10/30 Write block 475 len 32768 data: 3 3 3 2 2 3 3 2 11/30 Write block 557 len 24576 data: 6 6 7 4 3 3 12/30 Read block 67 len 36864 13/30 Read block 478 len 4096 14/30 Write block 55 len 36864 data: 4 3 3 4 3 3 3 3 2 15/30 Read block 315 len 12288 16/30 Write block 523 len 40960 data: 7 6 4 4 4 4 4 3 3 6 17/30 Read block 242 len 24576 18/30 Read block 584 len 20480 19/30 Read block 347 len 16384 20/30 Read block 542 len 20480 21/30 Flush 22/30 Write block 342 len 32768 data: 3 4 4 4 4 4 3 3 23/30 Write block 518 len 32768 data: 4 6 6 7 7 8 7 5 24/30 Write block 109 len 20480 data: 2 2 2 3 4 25/30 Write block 229 len 24576 data: 4 4 4 3 4 4 26/30 Write block 558 len 36864 data: 7 8 5 4 4 2 3 3 3 27/30 Read block 338 len 32768 28/30 Write block 360 len 4096 data: 4 29/30 Write block 478 len 4096 data: 3 30/30 Write block 157 len 16384 data: 2 3 3 4 ---------------------------------------------------------------- Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 4 Done Done Done false 1003 Acked Read 4 Sent Done Done false 1004 Acked Read 6 Sent Done Done false 1005 Acked Read 8 Sent Done Done false 1006 Acked Write 2 Sent Done Done false 1007 Acked Write 4 Sent Done Done false 1008 Acked Write 3 Sent Done Done false 1009 Acked Write 8 Sent Done Done false 1010 Acked Write 6 Sent Done Done false 1011 Acked Read 9 Sent Done Done false 1012 Acked Read 1 Sent Done Done false 1013 Acked Write 9 Sent Done Done false 1014 Acked Read 3 Sent Done Done false 1015 Acked Write 10 Sent Done Done false 1016 Acked Read 6 Sent Done Done false 1017 Acked Read 5 Sent Done Done false 1018 Acked Read 4 Sent Done Done false 1019 Acked Read 5 Sent Done Done false 1020 Acked Flush 0 Sent Done Done false 1021 Acked Write 8 Sent Done Done false 1022 Acked Write 8 Sent Done Done false 1023 Acked Write 5 Sent Done Done false 1024 Acked Write 6 Sent Done Done false 1025 Acked Write 9 Sent Done Done false 1026 Acked Read 8 Sent Done Done false 1027 Acked Write 1 Sent Done Sent false 1028 Acked Write 1 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 27 2 3 32 Done 1 26 25 52 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1001)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:43:22.272084485Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1285} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:22.27225416Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1285,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 35 35 35 61 61 61 F F F 1 020-039 36 36 36 63 63 63 F F F 2 040-059 29 38 38 54 68 68 F F F <--- 3 060-079 29 38 38 54 68 68 F F F <--- 4 080-099 35 35 35 61 61 61 F F F 5 100-119 35 38 38 61 68 68 F T T <--- 6 120-139 35 35 35 61 61 61 F F F 7 140-159 37 37 37 65 65 65 F T T <--- 8 160-179 35 35 35 61 61 61 F T T <--- 9 180-199 28 28 28 53 53 53 F F F 10 200-219 36 36 36 63 63 63 F F F 11 220-239 36 36 36 64 64 64 F T T <--- 12 240-259 27 27 27 51 51 51 F F F 13 260-279 36 36 36 63 63 63 F F F 14 280-299 36 36 36 63 63 63 F F F 15 300-319 36 36 36 63 63 63 F F F 16 320-339 37 38 38 65 68 68 F F F <--- 17 340-359 37 37 37 65 65 65 F T T <--- 18 360-379 35 38 38 61 68 68 F T T <--- 19 380-399 33 33 33 59 59 59 F F F 20 400-419 36 36 36 64 64 64 F F F 21 420-439 36 36 36 63 63 63 F F F 22 440-459 31 31 31 57 57 57 F F F 23 460-479 32 38 38 58 68 68 F T T <--- 24 480-499 32 38 38 58 68 68 F F F <--- 25 500-519 36 38 38 62 68 68 T T T <--- 26 520-539 36 38 38 62 68 68 F T T <--- 27 540-559 36 38 38 64 68 68 F T T <--- 28 560-579 31 38 38 57 68 68 F T T <--- 29 580-599 30 30 30 55 55 55 F F F Max gen: 38, Max flush: 68 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 19 Verifying data now No change after verify Loop: 19 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 35 35 35 61 61 61 F F F 1 020-039 36 36 36 63 63 63 F F F 2 040-059 38 38 38 68 68 68 F F F 3 060-079 38 38 38 68 68 68 F F F 4 080-099 35 35 35 61 61 61 F F F 5 100-119 39 39 39 69 69 69 F F F 6 120-139 35 35 35 61 61 61 F F F 7 140-159 39 39 39 69 69 69 F F F 8 160-179 39 39 39 69 69 69 F F F 9 180-199 28 28 28 53 53 53 F F F 10 200-219 36 36 36 63 63 63 F F F 11 220-239 39 39 39 69 69 69 F F F 12 240-259 27 27 27 51 51 51 F F F 13 260-279 36 36 36 63 63 63 F F F 14 280-299 36 36 36 63 63 63 F F F 15 300-319 36 36 36 63 63 63 F F F 16 320-339 38 38 38 68 68 68 F F F 17 340-359 39 39 39 69 69 69 F F F 18 360-379 39 39 39 69 69 69 F F F 19 380-399 33 33 33 59 59 59 F F F 20 400-419 36 36 36 64 64 64 F F F 21 420-439 36 36 36 63 63 63 F F F 22 440-459 31 31 31 57 57 57 F F F 23 460-479 39 39 39 69 69 69 F F F 24 480-499 38 38 38 68 68 68 F F F 25 500-519 39 39 39 69 69 69 F F F 26 520-539 39 39 39 69 69 69 F F F 27 540-559 39 39 39 69 69 69 F F F 28 560-579 39 39 39 69 69 69 F F F 29 580-599 30 30 30 55 55 55 F F F Max gen: 39, Max flush: 69 Begin loop 20 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:26.528017374Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1293,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:43:26.535017852Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1293,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:26.535072731Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1293,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:26.541784137Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1293,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:26.5420485Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1293} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:26.542073789Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1293} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:26.542082209Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1293} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:26.542093698Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1293} {"msg":"Crucible c87b62e9-2511-4b5c-84ad-b3bed34aad1f has session id: a1d93dd1-8902-432a-b6dd-9c191b57eb45","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:26.54238093Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1293,"session_id":"a1d93dd1-8902-432a-b6dd-9c191b57eb45"} {"msg":"Upstairs opts: Upstairs UUID: c87b62e9-2511-4b5c-84ad-b3bed34aad1f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:26.542402599Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1293,"session_id":"a1d93dd1-8902-432a-b6dd-9c191b57eb45"} {"msg":"Crucible stats registered with UUID: c87b62e9-2511-4b5c-84ad-b3bed34aad1f","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:26.542412539Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1293,"session_id":"a1d93dd1-8902-432a-b6dd-9c191b57eb45"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:43:26.542421579Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1293,"session_id":"a1d93dd1-8902-432a-b6dd-9c191b57eb45"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:26.542559195Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1293,"":"io task","client":"0","":"downstairs","session_id":"a1d93dd1-8902-432a-b6dd-9c191b57eb45"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:26.542576964Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1293,"":"io task","client":"1","":"downstairs","session_id":"a1d93dd1-8902-432a-b6dd-9c191b57eb45"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:26.542587044Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1293,"":"io task","client":"2","":"downstairs","session_id":"a1d93dd1-8902-432a-b6dd-9c191b57eb45"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:26.542752669Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1293,"task":"crutest"} {"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:26.542766649Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1293} {"msg":"c87b62e9-2511-4b5c-84ad-b3bed34aad1f active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:26.542799388Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1293,"session_id":"a1d93dd1-8902-432a-b6dd-9c191b57eb45"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:26.542808748Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1293,"client":"0","":"downstairs","session_id":"a1d93dd1-8902-432a-b6dd-9c191b57eb45"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:26.542818517Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1293,"client":"1","":"downstairs","session_id":"a1d93dd1-8902-432a-b6dd-9c191b57eb45"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:26.542828117Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1293,"client":"2","":"downstairs","session_id":"a1d93dd1-8902-432a-b6dd-9c191b57eb45"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:26.542837557Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1293,"":"io task","client":"2","":"downstairs","session_id":"a1d93dd1-8902-432a-b6dd-9c191b57eb45"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:26.542847457Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1293,"":"io task","client":"0","":"downstairs","session_id":"a1d93dd1-8902-432a-b6dd-9c191b57eb45"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:26.542857296Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1293,"":"io task","client":"1","":"downstairs","session_id":"a1d93dd1-8902-432a-b6dd-9c191b57eb45"} {"msg":"ds_connection connected from Ok(127.0.0.1:34065)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:26.542896925Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1293,"":"io task","client":"0","":"downstairs","session_id":"a1d93dd1-8902-432a-b6dd-9c191b57eb45"} {"msg":"ds_connection connected from Ok(127.0.0.1:54264)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:26.542936224Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1293,"":"io task","client":"2","":"downstairs","session_id":"a1d93dd1-8902-432a-b6dd-9c191b57eb45"} {"msg":"ds_connection connected from Ok(127.0.0.1:52898)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:26.54306959Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1293,"":"io task","client":"1","":"downstairs","session_id":"a1d93dd1-8902-432a-b6dd-9c191b57eb45"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:26.543624134Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1293,"client":"1","":"downstairs","session_id":"a1d93dd1-8902-432a-b6dd-9c191b57eb45"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:26.543641604Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1293,"client":"2","":"downstairs","session_id":"a1d93dd1-8902-432a-b6dd-9c191b57eb45"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:26.543723021Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1293,"client":"0","":"downstairs","session_id":"a1d93dd1-8902-432a-b6dd-9c191b57eb45"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:26.543938015Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1293,"session_id":"a1d93dd1-8902-432a-b6dd-9c191b57eb45"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:26.543973324Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1293,"session_id":"a1d93dd1-8902-432a-b6dd-9c191b57eb45"} {"msg":"[0]R flush_numbers[0..12]: [61, 63, 68, 68, 61, 69, 61, 69, 69, 53, 63, 69]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:26.544021163Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1293,"":"downstairs","":"downstairs","session_id":"a1d93dd1-8902-432a-b6dd-9c191b57eb45"} {"msg":"[0]R generation[0..12]: [35, 36, 38, 38, 35, 39, 35, 39, 39, 28, 36, 39]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:26.544032192Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1293,"":"downstairs","":"downstairs","session_id":"a1d93dd1-8902-432a-b6dd-9c191b57eb45"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:26.544043112Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1293,"":"downstairs","":"downstairs","session_id":"a1d93dd1-8902-432a-b6dd-9c191b57eb45"} {"msg":"[1]R flush_numbers[0..12]: [61, 63, 68, 68, 61, 69, 61, 69, 69, 53, 63, 69]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:26.544053552Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1293,"":"downstairs","":"downstairs","session_id":"a1d93dd1-8902-432a-b6dd-9c191b57eb45"} {"msg":"[1]R generation[0..12]: [35, 36, 38, 38, 35, 39, 35, 39, 39, 28, 36, 39]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:26.544063871Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1293,"":"downstairs","":"downstairs","session_id":"a1d93dd1-8902-432a-b6dd-9c191b57eb45"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:26.544074571Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1293,"":"downstairs","":"downstairs","session_id":"a1d93dd1-8902-432a-b6dd-9c191b57eb45"} {"msg":"[2]R flush_numbers[0..12]: [61, 63, 68, 68, 61, 69, 61, 69, 69, 53, 63, 69]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:26.544084731Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1293,"":"downstairs","":"downstairs","session_id":"a1d93dd1-8902-432a-b6dd-9c191b57eb45"} {"msg":"[2]R generation[0..12]: [35, 36, 38, 38, 35, 39, 35, 39, 39, 28, 36, 39]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:26.544095061Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1293,"":"downstairs","":"downstairs","session_id":"a1d93dd1-8902-432a-b6dd-9c191b57eb45"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:26.54410583Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1293,"":"downstairs","":"downstairs","session_id":"a1d93dd1-8902-432a-b6dd-9c191b57eb45"} {"msg":"Max found gen is 40","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:26.54411529Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1293,"":"downstairs","":"downstairs","session_id":"a1d93dd1-8902-432a-b6dd-9c191b57eb45"} {"msg":"Generation requested: 40 >= found:40","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:26.54412492Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1293,"":"downstairs","":"downstairs","session_id":"a1d93dd1-8902-432a-b6dd-9c191b57eb45"} {"msg":"Next flush: 70","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:26.544134249Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1293,"":"downstairs","":"downstairs","session_id":"a1d93dd1-8902-432a-b6dd-9c191b57eb45"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:26.544143479Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1293,"":"downstairs","":"downstairs","session_id":"a1d93dd1-8902-432a-b6dd-9c191b57eb45"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:26.544153139Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1293,"session_id":"a1d93dd1-8902-432a-b6dd-9c191b57eb45"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:26.544163469Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1293,"client":"0","":"downstairs","session_id":"a1d93dd1-8902-432a-b6dd-9c191b57eb45"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:26.544173028Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1293,"client":"1","":"downstairs","session_id":"a1d93dd1-8902-432a-b6dd-9c191b57eb45"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:26.544182638Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1293,"client":"2","":"downstairs","session_id":"a1d93dd1-8902-432a-b6dd-9c191b57eb45"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:26.544192368Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1293,"session_id":"a1d93dd1-8902-432a-b6dd-9c191b57eb45"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:26.544201387Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1293,"session_id":"a1d93dd1-8902-432a-b6dd-9c191b57eb45"} {"msg":"c87b62e9-2511-4b5c-84ad-b3bed34aad1f is now active with session: a1d93dd1-8902-432a-b6dd-9c191b57eb45","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:26.544211267Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1293,"session_id":"a1d93dd1-8902-432a-b6dd-9c191b57eb45"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:26.544219867Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1293,"session_id":"a1d93dd1-8902-432a-b6dd-9c191b57eb45"} {"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:26.544228787Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1293} {"msg":"Activated sub_volume c87b62e9-2511-4b5c-84ad-b3bed34aad1f","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:26.544236966Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1293} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:26.544245116Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1293,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:26.544576527Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1293,"task":"crutest"} 01/30 Write block 256 len 4096 data: 4 02/30 Write block 297 len 4096 data: 7 03/30 Write block 239 len 40960 data: 4 4 4 3 2 2 2 4 4 5 04/30 Read block 308 len 16384 05/30 Flush 06/30 Write block 535 len 40960 data: 8 5 5 5 5 5 3 3 3 5 07/30 Read block 141 len 16384 08/30 Read block 197 len 24576 09/30 Write block 558 len 16384 data: 8 9 6 5 10/30 Read block 149 len 12288 11/30 Read block 166 len 20480 12/30 Read block 206 len 20480 13/30 Read block 221 len 4096 14/30 Write block 321 len 12288 data: 3 3 4 15/30 Read block 411 len 36864 16/30 Flush 17/30 Write block 479 len 36864 data: 3 4 4 3 5 5 6 6 6 18/30 Write block 131 len 16384 data: 4 5 6 6 19/30 Read block 306 len 28672 20/30 Read block 82 len 20480 21/30 Write block 94 len 16384 data: 4 5 5 4 22/30 Read block 174 len 32768 23/30 Write block 139 len 16384 data: 5 4 4 4 24/30 Write block 244 len 16384 data: 3 3 5 5 25/30 Read block 120 len 36864 26/30 Write block 440 len 8192 data: 5 5 27/30 Read block 437 len 40960 28/30 Read block 19 len 16384 29/30 Write block 133 len 40960 data: 7 7 6 7 5 5 6 5 5 5 30/30 Read block 4 len 16384 ---------------------------------------------------------------- Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Write 1 Sent Done Done false 1002 Acked Write 10 Sent Done Done false 1003 Acked Read 4 Sent Done Done false 1004 Acked Flush 0 Sent Done Done false 1005 Acked Write 10 Sent Done Done false 1006 Acked Read 4 Sent Done Done false 1007 Acked Read 6 Sent Done Done false 1008 Acked Write 4 Sent Done Done false 1009 Acked Read 3 Sent Done Done false 1010 Acked Read 5 Sent Done Done false 1011 Acked Read 5 Sent Done Done false 1012 Acked Read 1 Sent Done Done false 1013 Acked Write 3 Sent Done Done false 1014 Acked Read 9 Sent Done Done false 1015 Acked Flush 0 Sent Done Done false 1016 Acked Write 9 Sent Done Done false 1017 Acked Write 4 Sent Done Done false 1018 Acked Read 7 Sent Done Done false 1019 Acked Read 5 Sent Done Done false 1020 Acked Write 4 Sent Done Done false 1021 Acked Read 8 Sent Done Done false 1022 Acked Write 4 Sent Done Done false 1023 Acked Write 4 Sent Done Done false 1024 Acked Read 9 Sent Done Done false 1025 Acked Write 2 Sent Done Done false 1026 Acked Read 10 Sent Done Done false 1027 Acked Read 4 Sent Done Done false 1028 Acked Write 10 Sent Done Done false 1029 Acked Read 4 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 0 0 29 Done 1 30 30 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1015)) Some(JobId(1015)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:43:26.553635835Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1293} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:26.55381917Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1293,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 35 35 35 61 61 61 F F F 1 020-039 36 36 36 63 63 63 F F F 2 040-059 38 38 38 68 68 68 F F F 3 060-079 38 38 38 68 68 68 F F F 4 080-099 35 35 35 61 61 61 F T T <--- 5 100-119 39 39 39 69 69 69 F F F 6 120-139 35 35 35 61 61 61 F T T <--- 7 140-159 39 39 39 69 69 69 F T T <--- 8 160-179 39 39 39 69 69 69 F F F 9 180-199 28 28 28 53 53 53 F F F 10 200-219 36 36 36 63 63 63 F F F 11 220-239 39 40 40 69 70 70 F F F <--- 12 240-259 27 40 40 51 70 70 T T T <--- 13 260-279 36 36 36 63 63 63 F F F 14 280-299 36 40 40 63 70 70 F F F <--- 15 300-319 36 36 36 63 63 63 F F F 16 320-339 38 40 40 68 71 71 F F F <--- 17 340-359 39 39 39 69 69 69 F F F 18 360-379 39 39 39 69 69 69 F F F 19 380-399 33 33 33 59 59 59 F F F 20 400-419 36 36 36 64 64 64 F F F 21 420-439 36 36 36 63 63 63 F F F 22 440-459 31 31 31 57 57 57 F T T <--- 23 460-479 39 39 39 69 69 69 F T T <--- 24 480-499 38 38 38 68 68 68 F T T <--- 25 500-519 39 39 39 69 69 69 F F F 26 520-539 39 40 40 69 71 71 F F F <--- 27 540-559 39 40 40 69 71 71 F F F <--- 28 560-579 39 40 40 69 71 71 F F F <--- 29 580-599 30 30 30 55 55 55 F F F Max gen: 40, Max flush: 71 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 20 Verifying data now No change after verify Loop: 20 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 35 35 35 61 61 61 F F F 1 020-039 36 36 36 63 63 63 F F F 2 040-059 38 38 38 68 68 68 F F F 3 060-079 38 38 38 68 68 68 F F F 4 080-099 41 41 41 72 72 72 F F F 5 100-119 39 39 39 69 69 69 F F F 6 120-139 41 41 41 72 72 72 F F F 7 140-159 41 41 41 72 72 72 F F F 8 160-179 39 39 39 69 69 69 F F F 9 180-199 28 28 28 53 53 53 F F F 10 200-219 36 36 36 63 63 63 F F F 11 220-239 40 40 40 70 70 70 F F F 12 240-259 41 41 41 72 72 72 F F F 13 260-279 36 36 36 63 63 63 F F F 14 280-299 40 40 40 70 70 70 F F F 15 300-319 36 36 36 63 63 63 F F F 16 320-339 40 40 40 71 71 71 F F F 17 340-359 39 39 39 69 69 69 F F F 18 360-379 39 39 39 69 69 69 F F F 19 380-399 33 33 33 59 59 59 F F F 20 400-419 36 36 36 64 64 64 F F F 21 420-439 36 36 36 63 63 63 F F F 22 440-459 41 41 41 72 72 72 F F F 23 460-479 41 41 41 72 72 72 F F F 24 480-499 41 41 41 72 72 72 F F F 25 500-519 39 39 39 69 69 69 F F F 26 520-539 40 40 40 71 71 71 F F F 27 540-559 40 40 40 71 71 71 F F F 28 560-579 40 40 40 71 71 71 F F F 29 580-599 30 30 30 55 55 55 F F F Max gen: 41, Max flush: 72 Begin loop 21 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:30.805271778Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1301,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:43:30.812295796Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1301,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:30.812350454Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1301,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:30.819014762Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1301,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:30.819276144Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1301} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:30.819301273Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1301} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:30.819312453Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1301} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:30.819323883Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1301} {"msg":"Crucible 9bbc51d2-8f52-405b-a7dd-26c7ab821dda has session id: ec9102b7-62c0-4e9c-9191-4fc0cb71ecce","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:30.819672813Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1301,"session_id":"ec9102b7-62c0-4e9c-9191-4fc0cb71ecce"} {"msg":"Upstairs opts: Upstairs UUID: 9bbc51d2-8f52-405b-a7dd-26c7ab821dda, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:30.819701712Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1301,"session_id":"ec9102b7-62c0-4e9c-9191-4fc0cb71ecce"} {"msg":"Crucible stats registered with UUID: 9bbc51d2-8f52-405b-a7dd-26c7ab821dda","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:30.819712521Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1301,"session_id":"ec9102b7-62c0-4e9c-9191-4fc0cb71ecce"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:43:30.819721601Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1301,"session_id":"ec9102b7-62c0-4e9c-9191-4fc0cb71ecce"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:30.819855557Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1301,"":"io task","client":"0","":"downstairs","session_id":"ec9102b7-62c0-4e9c-9191-4fc0cb71ecce"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:30.819887036Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1301,"":"io task","client":"2","":"downstairs","session_id":"ec9102b7-62c0-4e9c-9191-4fc0cb71ecce"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:30.819898016Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1301,"":"io task","client":"1","":"downstairs","session_id":"ec9102b7-62c0-4e9c-9191-4fc0cb71ecce"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:30.820047602Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1301,"task":"crutest"} {"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:30.820057781Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1301} {"msg":"9bbc51d2-8f52-405b-a7dd-26c7ab821dda active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:30.820075761Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1301,"session_id":"ec9102b7-62c0-4e9c-9191-4fc0cb71ecce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:30.820084561Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1301,"client":"0","":"downstairs","session_id":"ec9102b7-62c0-4e9c-9191-4fc0cb71ecce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:30.82009457Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1301,"client":"1","":"downstairs","session_id":"ec9102b7-62c0-4e9c-9191-4fc0cb71ecce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:30.82010436Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1301,"client":"2","":"downstairs","session_id":"ec9102b7-62c0-4e9c-9191-4fc0cb71ecce"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:30.82011585Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1301,"":"io task","client":"2","":"downstairs","session_id":"ec9102b7-62c0-4e9c-9191-4fc0cb71ecce"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:30.82012574Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1301,"":"io task","client":"0","":"downstairs","session_id":"ec9102b7-62c0-4e9c-9191-4fc0cb71ecce"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:30.820139979Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1301,"":"io task","client":"1","":"downstairs","session_id":"ec9102b7-62c0-4e9c-9191-4fc0cb71ecce"} {"msg":"ds_connection connected from Ok(127.0.0.1:51233)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:30.820194628Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1301,"":"io task","client":"0","":"downstairs","session_id":"ec9102b7-62c0-4e9c-9191-4fc0cb71ecce"} {"msg":"ds_connection connected from Ok(127.0.0.1:52207)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:30.820220457Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1301,"":"io task","client":"1","":"downstairs","session_id":"ec9102b7-62c0-4e9c-9191-4fc0cb71ecce"} {"msg":"ds_connection connected from Ok(127.0.0.1:59416)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:30.820237836Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1301,"":"io task","client":"2","":"downstairs","session_id":"ec9102b7-62c0-4e9c-9191-4fc0cb71ecce"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:30.820911287Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1301,"client":"1","":"downstairs","session_id":"ec9102b7-62c0-4e9c-9191-4fc0cb71ecce"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:30.820934106Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1301,"client":"2","":"downstairs","session_id":"ec9102b7-62c0-4e9c-9191-4fc0cb71ecce"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:30.820952476Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1301,"client":"0","":"downstairs","session_id":"ec9102b7-62c0-4e9c-9191-4fc0cb71ecce"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:30.821237867Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1301,"session_id":"ec9102b7-62c0-4e9c-9191-4fc0cb71ecce"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:30.821261947Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1301,"session_id":"ec9102b7-62c0-4e9c-9191-4fc0cb71ecce"} {"msg":"[0]R flush_numbers[0..12]: [61, 63, 68, 68, 72, 69, 72, 72, 69, 53, 63, 70]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:30.821306425Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1301,"":"downstairs","":"downstairs","session_id":"ec9102b7-62c0-4e9c-9191-4fc0cb71ecce"} {"msg":"[0]R generation[0..12]: [35, 36, 38, 38, 41, 39, 41, 41, 39, 28, 36, 40]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:30.821317765Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1301,"":"downstairs","":"downstairs","session_id":"ec9102b7-62c0-4e9c-9191-4fc0cb71ecce"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:30.821328775Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1301,"":"downstairs","":"downstairs","session_id":"ec9102b7-62c0-4e9c-9191-4fc0cb71ecce"} {"msg":"[1]R flush_numbers[0..12]: [61, 63, 68, 68, 72, 69, 72, 72, 69, 53, 63, 70]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:30.821341924Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1301,"":"downstairs","":"downstairs","session_id":"ec9102b7-62c0-4e9c-9191-4fc0cb71ecce"} {"msg":"[1]R generation[0..12]: [35, 36, 38, 38, 41, 39, 41, 41, 39, 28, 36, 40]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:30.821352404Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1301,"":"downstairs","":"downstairs","session_id":"ec9102b7-62c0-4e9c-9191-4fc0cb71ecce"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:30.821363194Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1301,"":"downstairs","":"downstairs","session_id":"ec9102b7-62c0-4e9c-9191-4fc0cb71ecce"} {"msg":"[2]R flush_numbers[0..12]: [61, 63, 68, 68, 72, 69, 72, 72, 69, 53, 63, 70]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:30.821373464Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1301,"":"downstairs","":"downstairs","session_id":"ec9102b7-62c0-4e9c-9191-4fc0cb71ecce"} {"msg":"[2]R generation[0..12]: [35, 36, 38, 38, 41, 39, 41, 41, 39, 28, 36, 40]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:30.821383683Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1301,"":"downstairs","":"downstairs","session_id":"ec9102b7-62c0-4e9c-9191-4fc0cb71ecce"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:30.821399593Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1301,"":"downstairs","":"downstairs","session_id":"ec9102b7-62c0-4e9c-9191-4fc0cb71ecce"} {"msg":"Max found gen is 42","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:30.821408842Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1301,"":"downstairs","":"downstairs","session_id":"ec9102b7-62c0-4e9c-9191-4fc0cb71ecce"} {"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:30.821418492Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1301,"":"downstairs","":"downstairs","session_id":"ec9102b7-62c0-4e9c-9191-4fc0cb71ecce"} {"msg":"Next flush: 73","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:30.821427752Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1301,"":"downstairs","":"downstairs","session_id":"ec9102b7-62c0-4e9c-9191-4fc0cb71ecce"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:30.821437132Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1301,"":"downstairs","":"downstairs","session_id":"ec9102b7-62c0-4e9c-9191-4fc0cb71ecce"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:30.821446741Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1301,"session_id":"ec9102b7-62c0-4e9c-9191-4fc0cb71ecce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:30.821455421Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1301,"client":"0","":"downstairs","session_id":"ec9102b7-62c0-4e9c-9191-4fc0cb71ecce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:30.821465141Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1301,"client":"1","":"downstairs","session_id":"ec9102b7-62c0-4e9c-9191-4fc0cb71ecce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:30.821474881Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1301,"client":"2","":"downstairs","session_id":"ec9102b7-62c0-4e9c-9191-4fc0cb71ecce"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:30.82148457Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1301,"session_id":"ec9102b7-62c0-4e9c-9191-4fc0cb71ecce"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:30.82149386Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1301,"session_id":"ec9102b7-62c0-4e9c-9191-4fc0cb71ecce"} {"msg":"9bbc51d2-8f52-405b-a7dd-26c7ab821dda is now active with session: ec9102b7-62c0-4e9c-9191-4fc0cb71ecce","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:30.82150537Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1301,"session_id":"ec9102b7-62c0-4e9c-9191-4fc0cb71ecce"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:30.821513999Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1301,"session_id":"ec9102b7-62c0-4e9c-9191-4fc0cb71ecce"} {"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:30.821522969Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1301} {"msg":"Activated sub_volume 9bbc51d2-8f52-405b-a7dd-26c7ab821dda","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:30.821530939Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1301} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:30.821538979Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1301,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:30.82185539Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1301,"task":"crutest"} 01/30 Write block 457 len 12288 data: 3 4 2 02/30 Read block 38 len 8192 03/30 Write block 271 len 12288 data: 5 5 6 04/30 Write block 404 len 4096 data: 6 05/30 Write block 61 len 28672 data: 4 4 3 2 2 2 2 06/30 Read block 388 len 4096 07/30 Read block 426 len 16384 08/30 Flush 09/30 Write block 278 len 32768 data: 3 3 2 2 3 2 2 3 10/30 Read block 179 len 24576 11/30 Read block 481 len 36864 12/30 Read block 386 len 20480 13/30 Write block 469 len 4096 data: 3 14/30 Read block 162 len 20480 15/30 Write block 34 len 4096 data: 4 16/30 Read block 527 len 32768 17/30 Write block 92 len 36864 data: 2 3 5 6 6 5 4 4 5 18/30 Read block 550 len 28672 19/30 Read block 527 len 28672 20/30 Write block 120 len 16384 data: 5 4 4 4 21/30 Write block 532 len 40960 data: 7 5 6 9 6 6 6 6 6 4 22/30 Write block 344 len 32768 data: 5 5 5 5 4 4 3 2 23/30 Read block 363 len 28672 24/30 Read block 175 len 32768 25/30 Read block 55 len 4096 26/30 Flush 27/30 Write block 175 len 40960 data: 4 4 4 3 2 2 2 2 2 3 28/30 Write block 476 len 28672 data: 4 4 4 4 5 5 4 29/30 Read block 9 len 8192 30/30 Read block 520 len 8192 ---------------------------------------------------------------- Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Read 2 Done Done Done false 1002 Acked Write 3 Done Done Done false 1003 Acked Write 1 Done Done Done false 1004 Acked Write 7 Done Done Done false 1005 Acked Read 1 Sent Done Done false 1006 Acked Read 4 Sent Done Done false 1007 Acked Flush 0 Sent Done Done false 1008 Acked Write 8 Sent Done Done false 1009 Acked Read 6 Sent Done Done false 1010 Acked Read 9 Sent Done Done false 1011 Acked Read 5 Sent Done Done false 1012 Acked Write 1 Sent Done Done false 1013 Acked Read 5 Sent Done Done false 1014 Acked Write 1 Sent Done Done false 1015 Acked Read 8 Sent Done Done false 1016 Acked Write 9 Sent Done Done false 1017 Acked Read 7 Sent Done Done false 1018 Acked Read 7 Sent Done Done false 1019 Acked Write 4 Sent Done Done false 1020 Acked Write 10 Sent Done Done false 1021 Acked Write 8 Sent Done Done false 1022 Acked Read 7 Sent Done Done false 1023 Acked Read 8 Sent Done Done false 1024 Acked Read 1 Sent Done Done false 1025 Acked Flush 0 Sent Done Done false 1026 Acked Write 10 Sent Done Done false 1027 Acked Write 7 Sent Done Done false 1028 Acked Read 2 Sent Done Done false 1029 Acked Read 2 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 25 0 0 25 Done 5 30 30 65 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:43:30.831013365Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1301} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:30.831180251Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1301,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 35 35 35 61 61 61 F F F 1 020-039 36 42 42 63 74 74 F F F <--- 2 040-059 38 38 38 68 68 68 F F F 3 060-079 38 42 42 68 73 73 T F F <--- 4 080-099 41 42 42 72 74 74 F F F <--- 5 100-119 39 42 42 69 74 74 F F F <--- 6 120-139 41 42 42 72 74 74 F F F <--- 7 140-159 41 41 41 72 72 72 F F F 8 160-179 39 39 39 69 69 69 F T T <--- 9 180-199 28 28 28 53 53 53 F T T <--- 10 200-219 36 36 36 63 63 63 F F F 11 220-239 40 40 40 70 70 70 F F F 12 240-259 41 41 41 72 72 72 F F F 13 260-279 36 42 42 63 74 74 T F F <--- 14 280-299 40 42 42 70 74 74 F F F <--- 15 300-319 36 36 36 63 63 63 F F F 16 320-339 40 40 40 71 71 71 F F F 17 340-359 39 42 42 69 74 74 F F F <--- 18 360-379 39 39 39 69 69 69 F F F 19 380-399 33 33 33 59 59 59 F F F 20 400-419 36 42 42 64 73 73 T F F <--- 21 420-439 36 36 36 63 63 63 F F F 22 440-459 41 42 42 72 73 73 T F F <--- 23 460-479 41 42 42 72 74 74 F T T <--- 24 480-499 41 41 41 72 72 72 F T T <--- 25 500-519 39 39 39 69 69 69 F F F 26 520-539 40 42 42 71 74 74 F F F <--- 27 540-559 40 42 42 71 74 74 F F F <--- 28 560-579 40 40 40 71 71 71 F F F 29 580-599 30 30 30 55 55 55 F F F Max gen: 42, Max flush: 74 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 21 Verifying data now No change after verify Loop: 21 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 35 35 35 61 61 61 F F F 1 020-039 42 42 42 74 74 74 F F F 2 040-059 38 38 38 68 68 68 F F F 3 060-079 42 42 42 73 73 73 F F F 4 080-099 42 42 42 74 74 74 F F F 5 100-119 42 42 42 74 74 74 F F F 6 120-139 42 42 42 74 74 74 F F F 7 140-159 41 41 41 72 72 72 F F F 8 160-179 43 43 43 75 75 75 F F F 9 180-199 43 43 43 75 75 75 F F F 10 200-219 36 36 36 63 63 63 F F F 11 220-239 40 40 40 70 70 70 F F F 12 240-259 41 41 41 72 72 72 F F F 13 260-279 42 42 42 74 74 74 F F F 14 280-299 42 42 42 74 74 74 F F F 15 300-319 36 36 36 63 63 63 F F F 16 320-339 40 40 40 71 71 71 F F F 17 340-359 42 42 42 74 74 74 F F F 18 360-379 39 39 39 69 69 69 F F F 19 380-399 33 33 33 59 59 59 F F F 20 400-419 42 42 42 73 73 73 F F F 21 420-439 36 36 36 63 63 63 F F F 22 440-459 42 42 42 73 73 73 F F F 23 460-479 43 43 43 75 75 75 F F F 24 480-499 43 43 43 75 75 75 F F F 25 500-519 39 39 39 69 69 69 F F F 26 520-539 42 42 42 74 74 74 F F F 27 540-559 42 42 42 74 74 74 F F F 28 560-579 40 40 40 71 71 71 F F F 29 580-599 30 30 30 55 55 55 F F F Max gen: 43, Max flush: 75 Begin loop 22 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:35.092826805Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1309,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:35.100926031Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1309,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:35.101204173Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1309} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:35.101232083Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1309} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:35.101240732Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1309} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:35.101257202Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1309} {"msg":"Crucible 701bb703-6b36-4f1e-acba-916a8ec918cf has session id: 9411d434-5c41-44ab-b66e-7b94ca92e286","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:35.101576203Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1309,"session_id":"9411d434-5c41-44ab-b66e-7b94ca92e286"} {"msg":"Upstairs opts: Upstairs UUID: 701bb703-6b36-4f1e-acba-916a8ec918cf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:35.101604122Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1309,"session_id":"9411d434-5c41-44ab-b66e-7b94ca92e286"} {"msg":"Crucible stats registered with UUID: 701bb703-6b36-4f1e-acba-916a8ec918cf","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:35.101614942Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1309,"session_id":"9411d434-5c41-44ab-b66e-7b94ca92e286"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:43:35.101626301Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1309,"session_id":"9411d434-5c41-44ab-b66e-7b94ca92e286"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:35.101742438Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1309,"":"io task","client":"0","":"downstairs","session_id":"9411d434-5c41-44ab-b66e-7b94ca92e286"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:35.101774297Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1309,"":"io task","client":"1","":"downstairs","session_id":"9411d434-5c41-44ab-b66e-7b94ca92e286"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:35.101784507Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1309,"":"io task","client":"2","":"downstairs","session_id":"9411d434-5c41-44ab-b66e-7b94ca92e286"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:35.101923713Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1309,"task":"crutest"} {"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:35.101935232Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1309} {"msg":"701bb703-6b36-4f1e-acba-916a8ec918cf active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:35.101946032Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1309,"session_id":"9411d434-5c41-44ab-b66e-7b94ca92e286"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:35.101954772Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1309,"client":"0","":"downstairs","session_id":"9411d434-5c41-44ab-b66e-7b94ca92e286"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:35.101964361Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1309,"client":"1","":"downstairs","session_id":"9411d434-5c41-44ab-b66e-7b94ca92e286"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:35.101973821Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1309,"client":"2","":"downstairs","session_id":"9411d434-5c41-44ab-b66e-7b94ca92e286"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:35.101983221Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1309,"":"io task","client":"2","":"downstairs","session_id":"9411d434-5c41-44ab-b66e-7b94ca92e286"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:35.101993111Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1309,"":"io task","client":"0","":"downstairs","session_id":"9411d434-5c41-44ab-b66e-7b94ca92e286"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:35.10200286Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1309,"":"io task","client":"1","":"downstairs","session_id":"9411d434-5c41-44ab-b66e-7b94ca92e286"} {"msg":"ds_connection connected from Ok(127.0.0.1:57228)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:35.102101127Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1309,"":"io task","client":"2","":"downstairs","session_id":"9411d434-5c41-44ab-b66e-7b94ca92e286"} {"msg":"ds_connection connected from Ok(127.0.0.1:62437)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:35.102125997Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1309,"":"io task","client":"1","":"downstairs","session_id":"9411d434-5c41-44ab-b66e-7b94ca92e286"} {"msg":"ds_connection connected from Ok(127.0.0.1:65009)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:35.102137116Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1309,"":"io task","client":"0","":"downstairs","session_id":"9411d434-5c41-44ab-b66e-7b94ca92e286"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:35.102755889Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1309,"client":"0","":"downstairs","session_id":"9411d434-5c41-44ab-b66e-7b94ca92e286"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:35.102788588Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1309,"client":"1","":"downstairs","session_id":"9411d434-5c41-44ab-b66e-7b94ca92e286"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:35.102800117Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1309,"client":"2","":"downstairs","session_id":"9411d434-5c41-44ab-b66e-7b94ca92e286"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:35.103079599Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1309,"session_id":"9411d434-5c41-44ab-b66e-7b94ca92e286"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:35.103089129Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1309,"session_id":"9411d434-5c41-44ab-b66e-7b94ca92e286"} {"msg":"[0]R flush_numbers[0..12]: [61, 74, 68, 73, 74, 74, 74, 72, 75, 75, 63, 70]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:35.103123158Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1309,"":"downstairs","":"downstairs","session_id":"9411d434-5c41-44ab-b66e-7b94ca92e286"} {"msg":"[0]R generation[0..12]: [35, 42, 38, 42, 42, 42, 42, 41, 43, 43, 36, 40]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:35.103134038Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1309,"":"downstairs","":"downstairs","session_id":"9411d434-5c41-44ab-b66e-7b94ca92e286"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:35.103144887Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1309,"":"downstairs","":"downstairs","session_id":"9411d434-5c41-44ab-b66e-7b94ca92e286"} {"msg":"[1]R flush_numbers[0..12]: [61, 74, 68, 73, 74, 74, 74, 72, 75, 75, 63, 70]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:35.103155187Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1309,"":"downstairs","":"downstairs","session_id":"9411d434-5c41-44ab-b66e-7b94ca92e286"} {"msg":"[1]R generation[0..12]: [35, 42, 38, 42, 42, 42, 42, 41, 43, 43, 36, 40]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:35.103165317Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1309,"":"downstairs","":"downstairs","session_id":"9411d434-5c41-44ab-b66e-7b94ca92e286"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:35.103176096Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1309,"":"downstairs","":"downstairs","session_id":"9411d434-5c41-44ab-b66e-7b94ca92e286"} {"msg":"[2]R flush_numbers[0..12]: [61, 74, 68, 73, 74, 74, 74, 72, 75, 75, 63, 70]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:35.103186366Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1309,"":"downstairs","":"downstairs","session_id":"9411d434-5c41-44ab-b66e-7b94ca92e286"} {"msg":"[2]R generation[0..12]: [35, 42, 38, 42, 42, 42, 42, 41, 43, 43, 36, 40]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:35.103196476Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1309,"":"downstairs","":"downstairs","session_id":"9411d434-5c41-44ab-b66e-7b94ca92e286"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:35.103207336Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1309,"":"downstairs","":"downstairs","session_id":"9411d434-5c41-44ab-b66e-7b94ca92e286"} {"msg":"Max found gen is 44","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:35.103218845Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1309,"":"downstairs","":"downstairs","session_id":"9411d434-5c41-44ab-b66e-7b94ca92e286"} {"msg":"Generation requested: 44 >= found:44","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:35.103228505Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1309,"":"downstairs","":"downstairs","session_id":"9411d434-5c41-44ab-b66e-7b94ca92e286"} {"msg":"Next flush: 76","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:35.103237845Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1309,"":"downstairs","":"downstairs","session_id":"9411d434-5c41-44ab-b66e-7b94ca92e286"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:35.103247204Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1309,"":"downstairs","":"downstairs","session_id":"9411d434-5c41-44ab-b66e-7b94ca92e286"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:35.103256764Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1309,"session_id":"9411d434-5c41-44ab-b66e-7b94ca92e286"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:35.103265344Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1309,"client":"0","":"downstairs","session_id":"9411d434-5c41-44ab-b66e-7b94ca92e286"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:35.103274854Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1309,"client":"1","":"downstairs","session_id":"9411d434-5c41-44ab-b66e-7b94ca92e286"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:35.103284323Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1309,"client":"2","":"downstairs","session_id":"9411d434-5c41-44ab-b66e-7b94ca92e286"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:35.103293993Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1309,"session_id":"9411d434-5c41-44ab-b66e-7b94ca92e286"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:35.103302923Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1309,"session_id":"9411d434-5c41-44ab-b66e-7b94ca92e286"} {"msg":"701bb703-6b36-4f1e-acba-916a8ec918cf is now active with session: 9411d434-5c41-44ab-b66e-7b94ca92e286","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:35.103312763Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1309,"session_id":"9411d434-5c41-44ab-b66e-7b94ca92e286"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:35.103321392Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1309,"session_id":"9411d434-5c41-44ab-b66e-7b94ca92e286"} {"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:35.103330242Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1309} {"msg":"Activated sub_volume 701bb703-6b36-4f1e-acba-916a8ec918cf","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:35.103338162Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1309} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:35.103346132Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1309,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:35.103603774Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1309,"task":"crutest"} 01/30 Write block 562 len 8192 data: 5 3 02/30 Flush 03/30 Write block 277 len 36864 data: 4 4 4 3 3 4 3 3 4 04/30 Read block 404 len 36864 05/30 Write block 375 len 16384 data: 5 6 6 6 06/30 Read block 294 len 12288 07/30 Read block 167 len 8192 08/30 Flush 09/30 Write block 151 len 24576 data: 3 3 2 2 2 3 10/30 Read block 589 len 24576 11/30 Write block 562 len 20480 data: 6 4 4 4 4 12/30 Write block 588 len 8192 data: 5 4 13/30 Write block 258 len 12288 data: 2 2 2 14/30 Read block 172 len 32768 15/30 Write block 141 len 24576 data: 6 6 5 5 6 7 16/30 Write block 204 len 4096 data: 8 17/30 Read block 462 len 8192 18/30 Write block 332 len 28672 data: 4 4 5 6 7 8 5 19/30 Write block 164 len 28672 data: 3 5 5 5 6 7 7 20/30 Read block 590 len 16384 21/30 Write block 489 len 8192 data: 3 3 22/30 Read block 546 len 12288 23/30 Read block 99 len 40960 24/30 Read block 88 len 24576 25/30 Write block 208 len 36864 data: 5 5 6 5 6 6 5 5 7 26/30 Write block 4 len 36864 data: 4 6 8 7 7 5 5 4 3 27/30 Write block 109 len 16384 data: 3 3 3 4 28/30 Read block 437 len 8192 29/30 Write block 122 len 28672 data: 5 5 4 3 4 4 4 30/30 Read block 493 len 32768 ---------------------------------------------------------------- Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 9 Done Done Sent false 1003 Acked Read 9 Done Done Sent false 1004 Acked Write 4 Done Done Sent false 1005 Acked Read 3 Done Done Sent false 1006 Acked Read 2 Done Done Sent false 1007 Acked Flush 0 Done Done Sent false 1008 Acked Write 6 Done Done Sent false 1009 Acked Read 6 Done Done Sent false 1010 Acked Write 5 Done Done Sent false 1011 Acked Write 2 Done Done Sent false 1012 Acked Write 3 Done Done Sent false 1013 Acked Read 8 Done Done Sent false 1014 Acked Write 6 Done Done Sent false 1015 Acked Write 1 Done Done Sent false 1016 Acked Read 2 Done Done Sent false 1017 Acked Write 7 Done Done Sent false 1018 Acked Write 7 Done Done Sent false 1019 Acked Read 4 Done Done Sent false 1020 Acked Write 2 Done Done Sent false 1021 Acked Read 3 Done Done Sent false 1022 Acked Read 10 Done Done Sent false 1023 Acked Read 6 Done Done Sent false 1024 Acked Write 9 Done Done Sent false 1025 Acked Write 9 Done Done Sent false 1026 Acked Write 4 Done Done Sent false 1027 Acked Read 2 Done Done Sent false 1028 Acked Write 7 Done Done Sent false 1029 Acked Read 8 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 28 28 Done 28 28 0 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1007)) Some(JobId(1007)) Some(JobId(1001)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:43:35.111668512Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1309} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:35.111843706Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1309,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 35 35 35 61 61 61 T T F <--- 1 020-039 42 42 42 74 74 74 F F F 2 040-059 38 38 38 68 68 68 F F F 3 060-079 42 42 42 73 73 73 F F F 4 080-099 42 42 42 74 74 74 F F F 5 100-119 42 42 42 74 74 74 T T F <--- 6 120-139 42 42 42 74 74 74 T T F <--- 7 140-159 41 41 41 72 72 72 T T F <--- 8 160-179 43 43 43 75 75 75 T T F <--- 9 180-199 43 43 43 75 75 75 F F F 10 200-219 36 36 36 63 63 63 T T F <--- 11 220-239 40 40 40 70 70 70 F F F 12 240-259 41 41 41 72 72 72 T T F <--- 13 260-279 44 44 42 77 77 74 T T F <--- 14 280-299 44 44 42 77 77 74 F F F <--- 15 300-319 36 36 36 63 63 63 F F F 16 320-339 40 40 40 71 71 71 T T F <--- 17 340-359 42 42 42 74 74 74 F F F 18 360-379 44 44 39 77 77 69 F F F <--- 19 380-399 33 33 33 59 59 59 F F F 20 400-419 42 42 42 73 73 73 F F F 21 420-439 36 36 36 63 63 63 F F F 22 440-459 42 42 42 73 73 73 F F F 23 460-479 43 43 43 75 75 75 F F F 24 480-499 43 43 43 75 75 75 T T F <--- 25 500-519 39 39 39 69 69 69 F F F 26 520-539 42 42 42 74 74 74 F F F 27 540-559 42 42 42 74 74 74 F F F 28 560-579 44 44 44 76 76 76 T T F <--- 29 580-599 30 30 30 55 55 55 T T F <--- Max gen: 44, Max flush: 77 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 22 Verifying data now No change after verify Loop: 22 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 45 45 45 78 78 78 F F F 1 020-039 42 42 42 74 74 74 F F F 2 040-059 38 38 38 68 68 68 F F F 3 060-079 42 42 42 73 73 73 F F F 4 080-099 42 42 42 74 74 74 F F F 5 100-119 45 45 45 78 78 78 F F F 6 120-139 45 45 45 78 78 78 F F F 7 140-159 45 45 45 78 78 78 F F F 8 160-179 45 45 45 78 78 78 F F F 9 180-199 43 43 43 75 75 75 F F F 10 200-219 45 45 45 78 78 78 F F F 11 220-239 40 40 40 70 70 70 F F F 12 240-259 45 45 45 78 78 78 F F F 13 260-279 45 45 45 78 78 78 F F F 14 280-299 44 44 44 77 77 77 F F F 15 300-319 36 36 36 63 63 63 F F F 16 320-339 45 45 45 78 78 78 F F F 17 340-359 42 42 42 74 74 74 F F F 18 360-379 44 44 44 77 77 77 F F F 19 380-399 33 33 33 59 59 59 F F F 20 400-419 42 42 42 73 73 73 F F F 21 420-439 36 36 36 63 63 63 F F F 22 440-459 42 42 42 73 73 73 F F F 23 460-479 43 43 43 75 75 75 F F F 24 480-499 45 45 45 78 78 78 F F F 25 500-519 39 39 39 69 69 69 F F F 26 520-539 42 42 42 74 74 74 F F F 27 540-559 42 42 42 74 74 74 F F F 28 560-579 45 45 45 78 78 78 F F F 29 580-599 45 45 45 78 78 78 F F F Max gen: 45, Max flush: 78 Begin loop 23 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:39.359529294Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1317,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:43:39.366494763Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1317,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:39.366549521Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1317,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:39.373199579Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1317,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:39.373464502Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1317} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:39.373490521Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1317} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:39.373499011Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1317} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:39.37351032Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1317} {"msg":"Crucible bc71d7fc-f794-4c9e-b055-bfb509be005a has session id: 4d436cb7-8511-4888-abdb-121c21d8731d","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:39.373905549Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1317,"session_id":"4d436cb7-8511-4888-abdb-121c21d8731d"} {"msg":"Upstairs opts: Upstairs UUID: bc71d7fc-f794-4c9e-b055-bfb509be005a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:39.373928518Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1317,"session_id":"4d436cb7-8511-4888-abdb-121c21d8731d"} {"msg":"Crucible stats registered with UUID: bc71d7fc-f794-4c9e-b055-bfb509be005a","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:39.373938608Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1317,"session_id":"4d436cb7-8511-4888-abdb-121c21d8731d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:43:39.373947678Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1317,"session_id":"4d436cb7-8511-4888-abdb-121c21d8731d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:39.374099953Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1317,"":"io task","client":"2","":"downstairs","session_id":"4d436cb7-8511-4888-abdb-121c21d8731d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:39.374117313Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1317,"":"io task","client":"0","":"downstairs","session_id":"4d436cb7-8511-4888-abdb-121c21d8731d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:39.374127512Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1317,"":"io task","client":"1","":"downstairs","session_id":"4d436cb7-8511-4888-abdb-121c21d8731d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:39.374258929Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1317,"task":"crutest"} {"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:39.374274308Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1317} {"msg":"bc71d7fc-f794-4c9e-b055-bfb509be005a active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:39.374307507Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1317,"session_id":"4d436cb7-8511-4888-abdb-121c21d8731d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:39.374316877Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1317,"client":"0","":"downstairs","session_id":"4d436cb7-8511-4888-abdb-121c21d8731d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:39.374326647Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1317,"client":"1","":"downstairs","session_id":"4d436cb7-8511-4888-abdb-121c21d8731d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:39.374336316Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1317,"client":"2","":"downstairs","session_id":"4d436cb7-8511-4888-abdb-121c21d8731d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:39.374345836Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1317,"":"io task","client":"2","":"downstairs","session_id":"4d436cb7-8511-4888-abdb-121c21d8731d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:39.374355866Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1317,"":"io task","client":"0","":"downstairs","session_id":"4d436cb7-8511-4888-abdb-121c21d8731d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:39.374365906Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1317,"":"io task","client":"1","":"downstairs","session_id":"4d436cb7-8511-4888-abdb-121c21d8731d"} {"msg":"ds_connection connected from Ok(127.0.0.1:38251)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:39.374447473Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1317,"":"io task","client":"2","":"downstairs","session_id":"4d436cb7-8511-4888-abdb-121c21d8731d"} {"msg":"ds_connection connected from Ok(127.0.0.1:51538)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:39.37454935Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1317,"":"io task","client":"1","":"downstairs","session_id":"4d436cb7-8511-4888-abdb-121c21d8731d"} {"msg":"ds_connection connected from Ok(127.0.0.1:61010)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:39.3745627Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1317,"":"io task","client":"0","":"downstairs","session_id":"4d436cb7-8511-4888-abdb-121c21d8731d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:39.375121954Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1317,"client":"1","":"downstairs","session_id":"4d436cb7-8511-4888-abdb-121c21d8731d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:39.375150133Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1317,"client":"0","":"downstairs","session_id":"4d436cb7-8511-4888-abdb-121c21d8731d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:39.375161003Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1317,"client":"2","":"downstairs","session_id":"4d436cb7-8511-4888-abdb-121c21d8731d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:39.375421455Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1317,"session_id":"4d436cb7-8511-4888-abdb-121c21d8731d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:39.375438245Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1317,"session_id":"4d436cb7-8511-4888-abdb-121c21d8731d"} {"msg":"[0]R flush_numbers[0..12]: [78, 74, 68, 73, 74, 78, 78, 78, 78, 75, 78, 70]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:39.375495533Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1317,"":"downstairs","":"downstairs","session_id":"4d436cb7-8511-4888-abdb-121c21d8731d"} {"msg":"[0]R generation[0..12]: [45, 42, 38, 42, 42, 45, 45, 45, 45, 43, 45, 40]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:39.375506543Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1317,"":"downstairs","":"downstairs","session_id":"4d436cb7-8511-4888-abdb-121c21d8731d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:39.375517382Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1317,"":"downstairs","":"downstairs","session_id":"4d436cb7-8511-4888-abdb-121c21d8731d"} {"msg":"[1]R flush_numbers[0..12]: [78, 74, 68, 73, 74, 78, 78, 78, 78, 75, 78, 70]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:39.375527832Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1317,"":"downstairs","":"downstairs","session_id":"4d436cb7-8511-4888-abdb-121c21d8731d"} {"msg":"[1]R generation[0..12]: [45, 42, 38, 42, 42, 45, 45, 45, 45, 43, 45, 40]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:39.375538032Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1317,"":"downstairs","":"downstairs","session_id":"4d436cb7-8511-4888-abdb-121c21d8731d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:39.375548821Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1317,"":"downstairs","":"downstairs","session_id":"4d436cb7-8511-4888-abdb-121c21d8731d"} {"msg":"[2]R flush_numbers[0..12]: [78, 74, 68, 73, 74, 78, 78, 78, 78, 75, 78, 70]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:39.375559141Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1317,"":"downstairs","":"downstairs","session_id":"4d436cb7-8511-4888-abdb-121c21d8731d"} {"msg":"[2]R generation[0..12]: [45, 42, 38, 42, 42, 45, 45, 45, 45, 43, 45, 40]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:39.375569341Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1317,"":"downstairs","":"downstairs","session_id":"4d436cb7-8511-4888-abdb-121c21d8731d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:39.375580121Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1317,"":"downstairs","":"downstairs","session_id":"4d436cb7-8511-4888-abdb-121c21d8731d"} {"msg":"Max found gen is 46","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:39.37558944Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1317,"":"downstairs","":"downstairs","session_id":"4d436cb7-8511-4888-abdb-121c21d8731d"} {"msg":"Generation requested: 46 >= found:46","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:39.37559921Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1317,"":"downstairs","":"downstairs","session_id":"4d436cb7-8511-4888-abdb-121c21d8731d"} {"msg":"Next flush: 79","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:39.37560852Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1317,"":"downstairs","":"downstairs","session_id":"4d436cb7-8511-4888-abdb-121c21d8731d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:39.375617899Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1317,"":"downstairs","":"downstairs","session_id":"4d436cb7-8511-4888-abdb-121c21d8731d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:39.375629069Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1317,"session_id":"4d436cb7-8511-4888-abdb-121c21d8731d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:39.375637679Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1317,"client":"0","":"downstairs","session_id":"4d436cb7-8511-4888-abdb-121c21d8731d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:39.375647449Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1317,"client":"1","":"downstairs","session_id":"4d436cb7-8511-4888-abdb-121c21d8731d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:39.375657398Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1317,"client":"2","":"downstairs","session_id":"4d436cb7-8511-4888-abdb-121c21d8731d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:39.375667508Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1317,"session_id":"4d436cb7-8511-4888-abdb-121c21d8731d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:39.375676678Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1317,"session_id":"4d436cb7-8511-4888-abdb-121c21d8731d"} {"msg":"bc71d7fc-f794-4c9e-b055-bfb509be005a is now active with session: 4d436cb7-8511-4888-abdb-121c21d8731d","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:39.375686548Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1317,"session_id":"4d436cb7-8511-4888-abdb-121c21d8731d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:39.375695207Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1317,"session_id":"4d436cb7-8511-4888-abdb-121c21d8731d"} {"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:39.375704247Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1317} {"msg":"Activated sub_volume bc71d7fc-f794-4c9e-b055-bfb509be005a","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:39.375712257Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1317} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:39.375720317Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1317,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:39.376065027Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1317,"task":"crutest"} 01/30 Write block 476 len 24576 data: 5 5 5 5 6 6 02/30 Read block 287 len 20480 03/30 Write block 536 len 32768 data: 7 7 7 7 7 5 4 4 04/30 Write block 194 len 4096 data: 3 05/30 Write block 89 len 32768 data: 3 3 2 3 4 6 7 7 06/30 Write block 100 len 24576 data: 6 2 2 2 3 2 07/30 Read block 294 len 32768 08/30 Read block 180 len 8192 09/30 Read block 263 len 32768 10/30 Read block 305 len 4096 11/30 Write block 495 len 8192 data: 4 4 12/30 Read block 310 len 8192 13/30 Read block 61 len 4096 14/30 Read block 344 len 36864 15/30 Write block 496 len 4096 data: 5 16/30 Read block 125 len 28672 17/30 Write block 564 len 32768 data: 5 5 5 2 2 2 3 3 18/30 Write block 141 len 4096 data: 7 19/30 Write block 127 len 20480 data: 5 5 3 3 5 20/30 Read block 415 len 28672 21/30 Read block 357 len 36864 22/30 Read block 402 len 8192 23/30 Write block 34 len 8192 data: 5 4 24/30 Flush 25/30 Write block 205 len 40960 data: 7 7 5 6 6 7 6 7 7 6 26/30 Read block 521 len 24576 27/30 Write block 12 len 28672 data: 4 4 4 4 5 6 6 28/30 Write block 475 len 8192 data: 4 6 29/30 Write block 273 len 36864 data: 7 5 4 4 5 5 5 4 4 30/30 Write block 141 len 16384 data: 8 7 6 6 ---------------------------------------------------------------- Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Read 5 Sent Done Done false 1002 Acked Write 8 Sent Done Done false 1003 Acked Write 1 Sent Done Done false 1004 Acked Write 8 Sent Done Done false 1005 Acked Write 6 Sent Done Done false 1006 Acked Read 8 Sent Done Done false 1007 Acked Read 2 Sent Done Done false 1008 Acked Read 8 Sent Done Done false 1009 Acked Read 1 Sent Done Done false 1010 Acked Write 2 Sent Done Done false 1011 Acked Read 2 Sent Done Done false 1012 Acked Read 1 Sent Done Done false 1013 Acked Read 9 Sent Done Done false 1014 Acked Write 1 Sent Done Done false 1015 Acked Read 7 Sent Done Done false 1016 Acked Write 8 Sent Done Done false 1017 Acked Write 1 Sent Done Done false 1018 Acked Write 5 Sent Done Done false 1019 Acked Read 7 Sent Done Done false 1020 Acked Read 9 Sent Done Done false 1021 Acked Read 2 Sent Done Done false 1022 Acked Write 2 Sent Done Done false 1023 Acked Flush 0 Sent Done Done false 1024 Acked Write 10 Sent Done Done false 1025 Acked Read 6 Sent Done Done false 1026 Acked Write 7 Sent Done Sent false 1027 Acked Write 2 Sent Sent Sent false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 3 4 36 Done 1 27 26 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:43:39.384315109Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1317} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:39.384494043Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1317,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 45 45 45 78 78 78 F T T <--- 1 020-039 42 46 46 74 79 79 F F F <--- 2 040-059 38 38 38 68 68 68 F F F 3 060-079 42 42 42 73 73 73 F F F 4 080-099 42 46 46 74 79 79 F F F <--- 5 100-119 45 46 46 78 79 79 F F F <--- 6 120-139 45 46 46 78 79 79 F F F <--- 7 140-159 45 46 46 78 79 79 F T T <--- 8 160-179 45 45 45 78 78 78 F F F 9 180-199 43 46 46 75 79 79 F F F <--- 10 200-219 45 45 45 78 78 78 F T T <--- 11 220-239 40 40 40 70 70 70 F F F 12 240-259 45 45 45 78 78 78 F F F 13 260-279 45 45 45 78 78 78 F T T <--- 14 280-299 44 44 44 77 77 77 F T T <--- 15 300-319 36 36 36 63 63 63 F F F 16 320-339 45 45 45 78 78 78 F F F 17 340-359 42 42 42 74 74 74 F F F 18 360-379 44 44 44 77 77 77 F F F 19 380-399 33 33 33 59 59 59 F F F 20 400-419 42 42 42 73 73 73 F F F 21 420-439 36 36 36 63 63 63 F F F 22 440-459 42 42 42 73 73 73 F F F 23 460-479 43 46 46 75 79 79 T T T <--- 24 480-499 45 46 46 78 79 79 T F F <--- 25 500-519 39 39 39 69 69 69 F F F 26 520-539 42 46 46 74 79 79 F F F <--- 27 540-559 42 46 46 74 79 79 F F F <--- 28 560-579 45 46 46 78 79 79 F F F <--- 29 580-599 45 45 45 78 78 78 F F F Max gen: 46, Max flush: 79 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 23 Verifying data now No change after verify Loop: 23 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 47 47 47 80 80 80 F F F 1 020-039 46 46 46 79 79 79 F F F 2 040-059 38 38 38 68 68 68 F F F 3 060-079 42 42 42 73 73 73 F F F 4 080-099 46 46 46 79 79 79 F F F 5 100-119 46 46 46 79 79 79 F F F 6 120-139 46 46 46 79 79 79 F F F 7 140-159 47 47 47 80 80 80 F F F 8 160-179 45 45 45 78 78 78 F F F 9 180-199 46 46 46 79 79 79 F F F 10 200-219 47 47 47 80 80 80 F F F 11 220-239 40 40 40 70 70 70 F F F 12 240-259 45 45 45 78 78 78 F F F 13 260-279 47 47 47 80 80 80 F F F 14 280-299 47 47 47 80 80 80 F F F 15 300-319 36 36 36 63 63 63 F F F 16 320-339 45 45 45 78 78 78 F F F 17 340-359 42 42 42 74 74 74 F F F 18 360-379 44 44 44 77 77 77 F F F 19 380-399 33 33 33 59 59 59 F F F 20 400-419 42 42 42 73 73 73 F F F 21 420-439 36 36 36 63 63 63 F F F 22 440-459 42 42 42 73 73 73 F F F 23 460-479 47 47 47 80 80 80 F F F 24 480-499 46 46 46 79 79 79 F F F 25 500-519 39 39 39 69 69 69 F F F 26 520-539 46 46 46 79 79 79 F F F 27 540-559 46 46 46 79 79 79 F F F 28 560-579 46 46 46 79 79 79 F F F 29 580-599 45 45 45 78 78 78 F F F Max gen: 47, Max flush: 80 Begin loop 24 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:43.644082347Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1325,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:43.652150854Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1325,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:43.652480545Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1325} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:43.652514094Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1325} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:43.652523464Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1325} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:43.652560383Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1325} {"msg":"Crucible 621f2f96-fe02-444c-bd5e-794674ca6186 has session id: 40600203-5c78-440a-b74c-4d679cca407b","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:43.652879223Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1325,"session_id":"40600203-5c78-440a-b74c-4d679cca407b"} {"msg":"Upstairs opts: Upstairs UUID: 621f2f96-fe02-444c-bd5e-794674ca6186, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:43.652899393Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1325,"session_id":"40600203-5c78-440a-b74c-4d679cca407b"} {"msg":"Crucible stats registered with UUID: 621f2f96-fe02-444c-bd5e-794674ca6186","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:43.652909723Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1325,"session_id":"40600203-5c78-440a-b74c-4d679cca407b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:43:43.652918772Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1325,"session_id":"40600203-5c78-440a-b74c-4d679cca407b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:43.653072518Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1325,"":"io task","client":"1","":"downstairs","session_id":"40600203-5c78-440a-b74c-4d679cca407b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:43.653090487Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1325,"":"io task","client":"0","":"downstairs","session_id":"40600203-5c78-440a-b74c-4d679cca407b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:43.653100487Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1325,"":"io task","client":"2","":"downstairs","session_id":"40600203-5c78-440a-b74c-4d679cca407b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:43.653235003Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1325,"task":"crutest"} {"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:43.653246803Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1325} {"msg":"621f2f96-fe02-444c-bd5e-794674ca6186 active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:43.653258233Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1325,"session_id":"40600203-5c78-440a-b74c-4d679cca407b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:43.653266832Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1325,"client":"0","":"downstairs","session_id":"40600203-5c78-440a-b74c-4d679cca407b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:43.653276672Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1325,"client":"1","":"downstairs","session_id":"40600203-5c78-440a-b74c-4d679cca407b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:43.653286312Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1325,"client":"2","":"downstairs","session_id":"40600203-5c78-440a-b74c-4d679cca407b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:43.653295891Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1325,"":"io task","client":"2","":"downstairs","session_id":"40600203-5c78-440a-b74c-4d679cca407b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:43.653308361Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1325,"":"io task","client":"0","":"downstairs","session_id":"40600203-5c78-440a-b74c-4d679cca407b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:43.65333252Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1325,"":"io task","client":"1","":"downstairs","session_id":"40600203-5c78-440a-b74c-4d679cca407b"} {"msg":"ds_connection connected from Ok(127.0.0.1:62965)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:43.653402808Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1325,"":"io task","client":"2","":"downstairs","session_id":"40600203-5c78-440a-b74c-4d679cca407b"} {"msg":"ds_connection connected from Ok(127.0.0.1:59390)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:43.653413938Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1325,"":"io task","client":"1","":"downstairs","session_id":"40600203-5c78-440a-b74c-4d679cca407b"} {"msg":"ds_connection connected from Ok(127.0.0.1:36183)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:43.653424388Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1325,"":"io task","client":"0","":"downstairs","session_id":"40600203-5c78-440a-b74c-4d679cca407b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:43.654081299Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1325,"client":"2","":"downstairs","session_id":"40600203-5c78-440a-b74c-4d679cca407b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:43.654124708Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1325,"client":"0","":"downstairs","session_id":"40600203-5c78-440a-b74c-4d679cca407b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:43.654136957Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1325,"client":"1","":"downstairs","session_id":"40600203-5c78-440a-b74c-4d679cca407b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:43.65440034Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1325,"session_id":"40600203-5c78-440a-b74c-4d679cca407b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:43.654431279Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1325,"session_id":"40600203-5c78-440a-b74c-4d679cca407b"} {"msg":"[0]R flush_numbers[0..12]: [80, 79, 68, 73, 79, 79, 79, 80, 78, 79, 80, 70]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:43.654473997Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1325,"":"downstairs","":"downstairs","session_id":"40600203-5c78-440a-b74c-4d679cca407b"} {"msg":"[0]R generation[0..12]: [47, 46, 38, 42, 46, 46, 46, 47, 45, 46, 47, 40]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:43.654485537Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1325,"":"downstairs","":"downstairs","session_id":"40600203-5c78-440a-b74c-4d679cca407b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:43.654496477Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1325,"":"downstairs","":"downstairs","session_id":"40600203-5c78-440a-b74c-4d679cca407b"} {"msg":"[1]R flush_numbers[0..12]: [80, 79, 68, 73, 79, 79, 79, 80, 78, 79, 80, 70]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:43.654506987Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1325,"":"downstairs","":"downstairs","session_id":"40600203-5c78-440a-b74c-4d679cca407b"} {"msg":"[1]R generation[0..12]: [47, 46, 38, 42, 46, 46, 46, 47, 45, 46, 47, 40]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:43.654519976Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1325,"":"downstairs","":"downstairs","session_id":"40600203-5c78-440a-b74c-4d679cca407b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:43.654530886Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1325,"":"downstairs","":"downstairs","session_id":"40600203-5c78-440a-b74c-4d679cca407b"} {"msg":"[2]R flush_numbers[0..12]: [80, 79, 68, 73, 79, 79, 79, 80, 78, 79, 80, 70]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:43.654541186Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1325,"":"downstairs","":"downstairs","session_id":"40600203-5c78-440a-b74c-4d679cca407b"} {"msg":"[2]R generation[0..12]: [47, 46, 38, 42, 46, 46, 46, 47, 45, 46, 47, 40]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:43.654551505Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1325,"":"downstairs","":"downstairs","session_id":"40600203-5c78-440a-b74c-4d679cca407b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:43.654562325Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1325,"":"downstairs","":"downstairs","session_id":"40600203-5c78-440a-b74c-4d679cca407b"} {"msg":"Max found gen is 48","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:43.654571685Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1325,"":"downstairs","":"downstairs","session_id":"40600203-5c78-440a-b74c-4d679cca407b"} {"msg":"Generation requested: 48 >= found:48","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:43.654581264Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1325,"":"downstairs","":"downstairs","session_id":"40600203-5c78-440a-b74c-4d679cca407b"} {"msg":"Next flush: 81","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:43.654590484Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1325,"":"downstairs","":"downstairs","session_id":"40600203-5c78-440a-b74c-4d679cca407b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:43.654599724Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1325,"":"downstairs","":"downstairs","session_id":"40600203-5c78-440a-b74c-4d679cca407b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:43.654609574Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1325,"session_id":"40600203-5c78-440a-b74c-4d679cca407b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:43.654618253Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1325,"client":"0","":"downstairs","session_id":"40600203-5c78-440a-b74c-4d679cca407b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:43.654628153Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1325,"client":"1","":"downstairs","session_id":"40600203-5c78-440a-b74c-4d679cca407b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:43.654638023Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1325,"client":"2","":"downstairs","session_id":"40600203-5c78-440a-b74c-4d679cca407b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:43.654648012Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1325,"session_id":"40600203-5c78-440a-b74c-4d679cca407b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:43.654657062Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1325,"session_id":"40600203-5c78-440a-b74c-4d679cca407b"} {"msg":"621f2f96-fe02-444c-bd5e-794674ca6186 is now active with session: 40600203-5c78-440a-b74c-4d679cca407b","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:43.654666922Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1325,"session_id":"40600203-5c78-440a-b74c-4d679cca407b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:43.654677162Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1325,"session_id":"40600203-5c78-440a-b74c-4d679cca407b"} {"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:43.654686161Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1325} {"msg":"Activated sub_volume 621f2f96-fe02-444c-bd5e-794674ca6186","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:43.654694171Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1325} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:43.654702241Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1325,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:43.655018272Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1325,"task":"crutest"} 01/30 Write block 224 len 36864 data: 7 7 7 6 5 5 5 5 4 02/30 Write block 170 len 40960 data: 8 7 5 5 4 5 5 5 4 3 03/30 Write block 356 len 20480 data: 4 4 4 3 5 04/30 Read block 111 len 4096 05/30 Write block 552 len 20480 data: 5 6 3 3 4 06/30 Read block 358 len 28672 07/30 Read block 510 len 32768 08/30 Read block 329 len 40960 09/30 Write block 121 len 36864 data: 5 6 6 5 4 5 6 6 4 10/30 Read block 483 len 12288 11/30 Read block 69 len 24576 12/30 Read block 324 len 12288 13/30 Read block 519 len 12288 14/30 Write block 292 len 36864 data: 4 5 6 6 7 8 6 5 5 15/30 Write block 314 len 4096 data: 4 16/30 Write block 130 len 28672 data: 4 6 6 8 8 7 8 17/30 Read block 404 len 20480 18/30 Write block 222 len 28672 data: 5 7 8 8 8 7 6 19/30 Write block 378 len 20480 data: 7 7 7 7 7 20/30 Write block 61 len 24576 data: 5 5 4 3 3 3 21/30 Write block 334 len 16384 data: 6 7 8 9 22/30 Write block 201 len 8192 data: 6 7 23/30 Read block 369 len 40960 24/30 Write block 591 len 20480 data: 5 4 3 4 5 25/30 Read block 315 len 32768 26/30 Write block 307 len 4096 data: 3 27/30 Read block 133 len 36864 28/30 Read block 450 len 12288 29/30 Read block 94 len 40960 30/30 Write block 379 len 4096 data: 8 ---------------------------------------------------------------- Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Write 10 Done Done Done false 1002 Acked Write 5 Done Done Done false 1003 Acked Read 1 Done Sent Done false 1004 Acked Write 5 Done Sent Done false 1005 Acked Read 7 Done Sent Done false 1006 Acked Read 8 Done Sent Done false 1007 Acked Read 10 Done Sent Done false 1008 Acked Write 9 Done Sent Done false 1009 Acked Read 3 Done Sent Done false 1010 Acked Read 6 Done Sent Done false 1011 Acked Read 3 Done Sent Done false 1012 Acked Read 3 Done Sent Done false 1013 Acked Write 9 Done Sent Done false 1014 Acked Write 1 Done Sent Done false 1015 Acked Write 7 Done Sent Done false 1016 Acked Read 5 Done Sent Done false 1017 Acked Write 7 Done Sent Done false 1018 Acked Write 5 Done Sent Done false 1019 Acked Write 6 Done Sent Done false 1020 Acked Write 4 Done Sent Done false 1021 Acked Write 2 Done Sent Done false 1022 Acked Read 10 Done Sent Done false 1023 Acked Write 5 Done Sent Done false 1024 Acked Read 8 Done Sent Done false 1025 Acked Write 1 Done Sent Done false 1026 Acked Read 9 Done Sent Done false 1027 Acked Read 3 Done Sent Done false 1028 Acked Read 10 Done Sent Done false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 27 1 29 Done 29 3 29 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:43:43.663244954Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1325} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:43.663426719Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1325,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 47 47 47 80 80 80 F F F 1 020-039 46 46 46 79 79 79 F F F 2 040-059 38 38 38 68 68 68 F F F 3 060-079 42 42 42 73 73 73 T F T <--- 4 080-099 46 46 46 79 79 79 F F F 5 100-119 46 46 46 79 79 79 F F F 6 120-139 46 46 46 79 79 79 T F T <--- 7 140-159 47 47 47 80 80 80 F F F 8 160-179 45 45 45 78 78 78 T T T 9 180-199 46 46 46 79 79 79 F F F 10 200-219 47 47 47 80 80 80 T F T <--- 11 220-239 40 40 40 70 70 70 T T T 12 240-259 45 45 45 78 78 78 F F F 13 260-279 47 47 47 80 80 80 F F F 14 280-299 47 47 47 80 80 80 T F T <--- 15 300-319 36 36 36 63 63 63 T F T <--- 16 320-339 45 45 45 78 78 78 T F T <--- 17 340-359 42 42 42 74 74 74 T T T 18 360-379 44 44 44 77 77 77 T T T 19 380-399 33 33 33 59 59 59 T F T <--- 20 400-419 42 42 42 73 73 73 F F F 21 420-439 36 36 36 63 63 63 F F F 22 440-459 42 42 42 73 73 73 F F F 23 460-479 47 47 47 80 80 80 F F F 24 480-499 46 46 46 79 79 79 F F F 25 500-519 39 39 39 69 69 69 F F F 26 520-539 46 46 46 79 79 79 F F F 27 540-559 46 46 46 79 79 79 T F T <--- 28 560-579 46 46 46 79 79 79 F F F 29 580-599 45 45 45 78 78 78 T F T <--- Max gen: 47, Max flush: 80 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 24 Verifying data now No change after verify Loop: 24 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 47 47 47 80 80 80 F F F 1 020-039 46 46 46 79 79 79 F F F 2 040-059 38 38 38 68 68 68 F F F 3 060-079 48 48 48 81 81 81 F F F 4 080-099 46 46 46 79 79 79 F F F 5 100-119 46 46 46 79 79 79 F F F 6 120-139 48 48 48 81 81 81 F F F 7 140-159 47 47 47 80 80 80 F F F 8 160-179 48 48 48 81 81 81 F F F 9 180-199 46 46 46 79 79 79 F F F 10 200-219 48 48 48 81 81 81 F F F 11 220-239 48 48 48 81 81 81 F F F 12 240-259 45 45 45 78 78 78 F F F 13 260-279 47 47 47 80 80 80 F F F 14 280-299 48 48 48 81 81 81 F F F 15 300-319 48 48 48 81 81 81 F F F 16 320-339 48 48 48 81 81 81 F F F 17 340-359 48 48 48 81 81 81 F F F 18 360-379 48 48 48 81 81 81 F F F 19 380-399 48 48 48 81 81 81 F F F 20 400-419 42 42 42 73 73 73 F F F 21 420-439 36 36 36 63 63 63 F F F 22 440-459 42 42 42 73 73 73 F F F 23 460-479 47 47 47 80 80 80 F F F 24 480-499 46 46 46 79 79 79 F F F 25 500-519 39 39 39 69 69 69 F F F 26 520-539 46 46 46 79 79 79 F F F 27 540-559 48 48 48 81 81 81 F F F 28 560-579 46 46 46 79 79 79 F F F 29 580-599 48 48 48 81 81 81 F F F Max gen: 48, Max flush: 81 Begin loop 25 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:47.912621332Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1333,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:47.920633231Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1333,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:47.920940842Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1333} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:47.920966282Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1333} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:47.920974871Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1333} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:47.920986491Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1333} {"msg":"Crucible 501f9b8c-d8f7-4f46-b930-8f86f0a36488 has session id: 9b053cb2-4e65-465a-af85-ff196164c045","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:47.92135641Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1333,"session_id":"9b053cb2-4e65-465a-af85-ff196164c045"} {"msg":"Upstairs opts: Upstairs UUID: 501f9b8c-d8f7-4f46-b930-8f86f0a36488, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:47.921385699Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1333,"session_id":"9b053cb2-4e65-465a-af85-ff196164c045"} {"msg":"Crucible stats registered with UUID: 501f9b8c-d8f7-4f46-b930-8f86f0a36488","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:47.921396959Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1333,"session_id":"9b053cb2-4e65-465a-af85-ff196164c045"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:43:47.921408499Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1333,"session_id":"9b053cb2-4e65-465a-af85-ff196164c045"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:47.921545945Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1333,"":"io task","client":"2","":"downstairs","session_id":"9b053cb2-4e65-465a-af85-ff196164c045"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:47.921577144Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1333,"":"io task","client":"0","":"downstairs","session_id":"9b053cb2-4e65-465a-af85-ff196164c045"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:47.921587944Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1333,"":"io task","client":"1","":"downstairs","session_id":"9b053cb2-4e65-465a-af85-ff196164c045"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:47.921677681Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1333,"task":"crutest"} {"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:47.921691621Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1333} {"msg":"501f9b8c-d8f7-4f46-b930-8f86f0a36488 active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:47.92172108Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1333,"session_id":"9b053cb2-4e65-465a-af85-ff196164c045"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:47.92173042Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1333,"client":"0","":"downstairs","session_id":"9b053cb2-4e65-465a-af85-ff196164c045"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:47.921740529Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1333,"client":"1","":"downstairs","session_id":"9b053cb2-4e65-465a-af85-ff196164c045"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:47.921750219Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1333,"client":"2","":"downstairs","session_id":"9b053cb2-4e65-465a-af85-ff196164c045"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:47.921759819Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1333,"":"io task","client":"2","":"downstairs","session_id":"9b053cb2-4e65-465a-af85-ff196164c045"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:47.921769708Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1333,"":"io task","client":"0","":"downstairs","session_id":"9b053cb2-4e65-465a-af85-ff196164c045"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:47.921779558Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1333,"":"io task","client":"1","":"downstairs","session_id":"9b053cb2-4e65-465a-af85-ff196164c045"} {"msg":"ds_connection connected from Ok(127.0.0.1:33439)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:47.921863926Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1333,"":"io task","client":"2","":"downstairs","session_id":"9b053cb2-4e65-465a-af85-ff196164c045"} {"msg":"ds_connection connected from Ok(127.0.0.1:47452)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:47.921932564Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1333,"":"io task","client":"1","":"downstairs","session_id":"9b053cb2-4e65-465a-af85-ff196164c045"} {"msg":"ds_connection connected from Ok(127.0.0.1:40354)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:47.921947653Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1333,"":"io task","client":"0","":"downstairs","session_id":"9b053cb2-4e65-465a-af85-ff196164c045"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:47.922573715Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1333,"client":"0","":"downstairs","session_id":"9b053cb2-4e65-465a-af85-ff196164c045"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:47.922600394Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1333,"client":"1","":"downstairs","session_id":"9b053cb2-4e65-465a-af85-ff196164c045"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:47.922611434Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1333,"client":"2","":"downstairs","session_id":"9b053cb2-4e65-465a-af85-ff196164c045"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:47.922892426Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1333,"session_id":"9b053cb2-4e65-465a-af85-ff196164c045"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:47.922902196Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1333,"session_id":"9b053cb2-4e65-465a-af85-ff196164c045"} {"msg":"[0]R flush_numbers[0..12]: [80, 79, 68, 81, 79, 79, 81, 80, 81, 79, 81, 81]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:47.922945585Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1333,"":"downstairs","":"downstairs","session_id":"9b053cb2-4e65-465a-af85-ff196164c045"} {"msg":"[0]R generation[0..12]: [47, 46, 38, 48, 46, 46, 48, 47, 48, 46, 48, 48]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:47.922956424Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1333,"":"downstairs","":"downstairs","session_id":"9b053cb2-4e65-465a-af85-ff196164c045"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:47.922967564Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1333,"":"downstairs","":"downstairs","session_id":"9b053cb2-4e65-465a-af85-ff196164c045"} {"msg":"[1]R flush_numbers[0..12]: [80, 79, 68, 81, 79, 79, 81, 80, 81, 79, 81, 81]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:47.922977914Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1333,"":"downstairs","":"downstairs","session_id":"9b053cb2-4e65-465a-af85-ff196164c045"} {"msg":"[1]R generation[0..12]: [47, 46, 38, 48, 46, 46, 48, 47, 48, 46, 48, 48]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:47.922988253Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1333,"":"downstairs","":"downstairs","session_id":"9b053cb2-4e65-465a-af85-ff196164c045"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:47.922999173Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1333,"":"downstairs","":"downstairs","session_id":"9b053cb2-4e65-465a-af85-ff196164c045"} {"msg":"[2]R flush_numbers[0..12]: [80, 79, 68, 81, 79, 79, 81, 80, 81, 79, 81, 81]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:47.923009513Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1333,"":"downstairs","":"downstairs","session_id":"9b053cb2-4e65-465a-af85-ff196164c045"} {"msg":"[2]R generation[0..12]: [47, 46, 38, 48, 46, 46, 48, 47, 48, 46, 48, 48]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:47.923019872Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1333,"":"downstairs","":"downstairs","session_id":"9b053cb2-4e65-465a-af85-ff196164c045"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:47.923030622Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1333,"":"downstairs","":"downstairs","session_id":"9b053cb2-4e65-465a-af85-ff196164c045"} {"msg":"Max found gen is 49","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:47.923039882Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1333,"":"downstairs","":"downstairs","session_id":"9b053cb2-4e65-465a-af85-ff196164c045"} {"msg":"Generation requested: 50 >= found:49","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:47.923050911Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1333,"":"downstairs","":"downstairs","session_id":"9b053cb2-4e65-465a-af85-ff196164c045"} {"msg":"Next flush: 82","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:47.923060201Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1333,"":"downstairs","":"downstairs","session_id":"9b053cb2-4e65-465a-af85-ff196164c045"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:47.923069451Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1333,"":"downstairs","":"downstairs","session_id":"9b053cb2-4e65-465a-af85-ff196164c045"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:47.923079501Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1333,"session_id":"9b053cb2-4e65-465a-af85-ff196164c045"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:47.92308836Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1333,"client":"0","":"downstairs","session_id":"9b053cb2-4e65-465a-af85-ff196164c045"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:47.92309835Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1333,"client":"1","":"downstairs","session_id":"9b053cb2-4e65-465a-af85-ff196164c045"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:47.92310801Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1333,"client":"2","":"downstairs","session_id":"9b053cb2-4e65-465a-af85-ff196164c045"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:47.92311776Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1333,"session_id":"9b053cb2-4e65-465a-af85-ff196164c045"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:47.923126859Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1333,"session_id":"9b053cb2-4e65-465a-af85-ff196164c045"} {"msg":"501f9b8c-d8f7-4f46-b930-8f86f0a36488 is now active with session: 9b053cb2-4e65-465a-af85-ff196164c045","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:47.923136619Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1333,"session_id":"9b053cb2-4e65-465a-af85-ff196164c045"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:47.923145339Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1333,"session_id":"9b053cb2-4e65-465a-af85-ff196164c045"} {"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:47.923154538Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1333} {"msg":"Activated sub_volume 501f9b8c-d8f7-4f46-b930-8f86f0a36488","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:47.923162388Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1333} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:47.923170428Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1333,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:47.923499039Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1333,"task":"crutest"} 01/30 Write block 221 len 36864 data: 4 6 8 9 9 9 8 7 6 02/30 Flush 03/30 Flush 04/30 Write block 568 len 4096 data: 3 05/30 Read block 33 len 20480 06/30 Read block 392 len 8192 07/30 Write block 322 len 24576 data: 4 5 5 5 4 4 08/30 Write block 283 len 24576 data: 4 4 5 3 3 2 09/30 Flush 10/30 Write block 90 len 20480 data: 4 3 4 5 7 11/30 Read block 547 len 16384 12/30 Read block 59 len 4096 13/30 Write block 477 len 36864 data: 6 6 6 7 7 5 6 6 7 14/30 Read block 561 len 20480 15/30 Read block 184 len 8192 16/30 Read block 182 len 36864 17/30 Read block 83 len 16384 18/30 Read block 500 len 24576 19/30 Write block 390 len 24576 data: 2 2 2 2 2 2 20/30 Read block 336 len 4096 21/30 Write block 184 len 16384 data: 4 3 3 3 22/30 Read block 241 len 40960 23/30 Read block 81 len 36864 24/30 Read block 105 len 36864 25/30 Write block 47 len 32768 data: 2 3 2 2 2 3 3 4 26/30 Read block 321 len 12288 27/30 Read block 65 len 32768 28/30 Read block 248 len 24576 29/30 Read block 463 len 4096 30/30 Read block 157 len 32768 ---------------------------------------------------------------- Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1003 Acked Write 1 Done Done Done false 1004 Acked Read 5 Done Done Done false 1005 Acked Read 2 Done Done Done false 1006 Acked Write 6 Done Done Sent false 1007 Acked Write 6 Done Done Sent false 1008 Acked Flush 0 Done Done Sent false 1009 Acked Write 5 Done Done Sent false 1010 Acked Read 4 Done Done Sent false 1011 Acked Read 1 Done Done Sent false 1012 Acked Write 9 Done Done Sent false 1013 Acked Read 5 Done Done Sent false 1014 Acked Read 2 Done Done Sent false 1015 Acked Read 9 Done Done Sent false 1016 Acked Read 4 Done Done Sent false 1017 Acked Read 6 Done Done Sent false 1018 Acked Write 6 Done Done Sent false 1019 Acked Read 1 Done Done Sent false 1020 Acked Write 4 Done Done Sent false 1021 Acked Read 10 Done Done Sent false 1022 Acked Read 9 Done Done Sent false 1023 Acked Read 9 Done Done Sent false 1024 Acked Write 8 Done Done Sent false 1025 Acked Read 3 Done Done Sent false 1026 Acked Read 8 Done Done Sent false 1027 Acked Read 6 Done Done Sent false 1028 Acked Read 1 Done Done Sent false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 24 25 Done 27 26 3 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1008)) Some(JobId(1008)) Some(JobId(1002)) Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:43:47.932741992Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1333} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:47.932920627Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1333,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 47 47 47 80 80 80 F F F 1 020-039 46 46 46 79 79 79 F F F 2 040-059 38 38 38 68 68 68 T T F <--- 3 060-079 48 48 48 81 81 81 F F F 4 080-099 46 46 46 79 79 79 T T F <--- 5 100-119 46 46 46 79 79 79 F F F 6 120-139 48 48 48 81 81 81 F F F 7 140-159 47 47 47 80 80 80 F F F 8 160-179 48 48 48 81 81 81 F F F 9 180-199 46 46 46 79 79 79 T T F <--- 10 200-219 48 48 48 81 81 81 F F F 11 220-239 50 50 50 82 82 82 F F F 12 240-259 45 45 45 78 78 78 F F F 13 260-279 47 47 47 80 80 80 F F F 14 280-299 50 50 48 84 84 81 F F F <--- 15 300-319 48 48 48 81 81 81 F F F 16 320-339 50 50 48 84 84 81 F F F <--- 17 340-359 48 48 48 81 81 81 F F F 18 360-379 48 48 48 81 81 81 F F F 19 380-399 48 48 48 81 81 81 T T F <--- 20 400-419 42 42 42 73 73 73 F F F 21 420-439 36 36 36 63 63 63 F F F 22 440-459 42 42 42 73 73 73 F F F 23 460-479 47 47 47 80 80 80 T T F <--- 24 480-499 46 46 46 79 79 79 T T F <--- 25 500-519 39 39 39 69 69 69 F F F 26 520-539 46 46 46 79 79 79 F F F 27 540-559 48 48 48 81 81 81 F F F 28 560-579 50 50 46 84 84 79 F F T <--- 29 580-599 48 48 48 81 81 81 F F F Max gen: 50, Max flush: 84 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 25 Verifying data now No change after verify Loop: 25 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 47 47 47 80 80 80 F F F 1 020-039 46 46 46 79 79 79 F F F 2 040-059 51 51 51 85 85 85 F F F 3 060-079 48 48 48 81 81 81 F F F 4 080-099 51 51 51 85 85 85 F F F 5 100-119 46 46 46 79 79 79 F F F 6 120-139 48 48 48 81 81 81 F F F 7 140-159 47 47 47 80 80 80 F F F 8 160-179 48 48 48 81 81 81 F F F 9 180-199 51 51 51 85 85 85 F F F 10 200-219 48 48 48 81 81 81 F F F 11 220-239 50 50 50 82 82 82 F F F 12 240-259 45 45 45 78 78 78 F F F 13 260-279 47 47 47 80 80 80 F F F 14 280-299 50 50 50 84 84 84 F F F 15 300-319 48 48 48 81 81 81 F F F 16 320-339 50 50 50 84 84 84 F F F 17 340-359 48 48 48 81 81 81 F F F 18 360-379 48 48 48 81 81 81 F F F 19 380-399 51 51 51 85 85 85 F F F 20 400-419 42 42 42 73 73 73 F F F 21 420-439 36 36 36 63 63 63 F F F 22 440-459 42 42 42 73 73 73 F F F 23 460-479 51 51 51 85 85 85 F F F 24 480-499 51 51 51 85 85 85 F F F 25 500-519 39 39 39 69 69 69 F F F 26 520-539 46 46 46 79 79 79 F F F 27 540-559 48 48 48 81 81 81 F F F 28 560-579 50 50 50 84 84 84 F F F 29 580-599 48 48 48 81 81 81 F F F Max gen: 51, Max flush: 85 Begin loop 26 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:52.173938306Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1341,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:43:52.180947554Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1341,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:52.180994303Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1341,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:52.187795347Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1341,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:52.188065079Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1341} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:52.188093678Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1341} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:52.188102178Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1341} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:52.188113707Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1341} {"msg":"Crucible db20afd4-c9f5-43f3-b81a-b0f956ac0242 has session id: 93402788-2fd3-4971-a5a9-5f7b4f31a615","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:52.188455597Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1341,"session_id":"93402788-2fd3-4971-a5a9-5f7b4f31a615"} {"msg":"Upstairs opts: Upstairs UUID: db20afd4-c9f5-43f3-b81a-b0f956ac0242, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:52.188485377Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1341,"session_id":"93402788-2fd3-4971-a5a9-5f7b4f31a615"} {"msg":"Crucible stats registered with UUID: db20afd4-c9f5-43f3-b81a-b0f956ac0242","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:52.188496526Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1341,"session_id":"93402788-2fd3-4971-a5a9-5f7b4f31a615"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:43:52.188505726Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1341,"session_id":"93402788-2fd3-4971-a5a9-5f7b4f31a615"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:52.188637242Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1341,"":"io task","client":"1","":"downstairs","session_id":"93402788-2fd3-4971-a5a9-5f7b4f31a615"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:52.188655312Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1341,"":"io task","client":"0","":"downstairs","session_id":"93402788-2fd3-4971-a5a9-5f7b4f31a615"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:52.188669761Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1341,"":"io task","client":"2","":"downstairs","session_id":"93402788-2fd3-4971-a5a9-5f7b4f31a615"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:52.188794778Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1341,"task":"crutest"} {"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:52.188810367Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1341} {"msg":"db20afd4-c9f5-43f3-b81a-b0f956ac0242 active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:52.188826947Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1341,"session_id":"93402788-2fd3-4971-a5a9-5f7b4f31a615"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:52.188838276Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1341,"client":"0","":"downstairs","session_id":"93402788-2fd3-4971-a5a9-5f7b4f31a615"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:52.188847936Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1341,"client":"1","":"downstairs","session_id":"93402788-2fd3-4971-a5a9-5f7b4f31a615"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:52.188857466Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1341,"client":"2","":"downstairs","session_id":"93402788-2fd3-4971-a5a9-5f7b4f31a615"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:52.188867046Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1341,"":"io task","client":"2","":"downstairs","session_id":"93402788-2fd3-4971-a5a9-5f7b4f31a615"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:52.188876985Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1341,"":"io task","client":"0","":"downstairs","session_id":"93402788-2fd3-4971-a5a9-5f7b4f31a615"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:52.188912874Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1341,"":"io task","client":"1","":"downstairs","session_id":"93402788-2fd3-4971-a5a9-5f7b4f31a615"} {"msg":"ds_connection connected from Ok(127.0.0.1:42819)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:52.188927914Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1341,"":"io task","client":"0","":"downstairs","session_id":"93402788-2fd3-4971-a5a9-5f7b4f31a615"} {"msg":"ds_connection connected from Ok(127.0.0.1:53350)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:52.188938594Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1341,"":"io task","client":"1","":"downstairs","session_id":"93402788-2fd3-4971-a5a9-5f7b4f31a615"} {"msg":"ds_connection connected from Ok(127.0.0.1:49258)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:52.188949003Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1341,"":"io task","client":"2","":"downstairs","session_id":"93402788-2fd3-4971-a5a9-5f7b4f31a615"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:52.189639483Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1341,"client":"2","":"downstairs","session_id":"93402788-2fd3-4971-a5a9-5f7b4f31a615"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:52.189714241Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1341,"client":"1","":"downstairs","session_id":"93402788-2fd3-4971-a5a9-5f7b4f31a615"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:52.189726561Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1341,"client":"0","":"downstairs","session_id":"93402788-2fd3-4971-a5a9-5f7b4f31a615"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:52.189960104Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1341,"session_id":"93402788-2fd3-4971-a5a9-5f7b4f31a615"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:52.189985303Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1341,"session_id":"93402788-2fd3-4971-a5a9-5f7b4f31a615"} {"msg":"[0]R flush_numbers[0..12]: [80, 79, 85, 81, 85, 79, 81, 80, 81, 85, 81, 82]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:52.190060731Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1341,"":"downstairs","":"downstairs","session_id":"93402788-2fd3-4971-a5a9-5f7b4f31a615"} {"msg":"[0]R generation[0..12]: [47, 46, 51, 48, 51, 46, 48, 47, 48, 51, 48, 50]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:52.190074511Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1341,"":"downstairs","":"downstairs","session_id":"93402788-2fd3-4971-a5a9-5f7b4f31a615"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:52.19008573Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1341,"":"downstairs","":"downstairs","session_id":"93402788-2fd3-4971-a5a9-5f7b4f31a615"} {"msg":"[1]R flush_numbers[0..12]: [80, 79, 85, 81, 85, 79, 81, 80, 81, 85, 81, 82]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:52.19009642Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1341,"":"downstairs","":"downstairs","session_id":"93402788-2fd3-4971-a5a9-5f7b4f31a615"} {"msg":"[1]R generation[0..12]: [47, 46, 51, 48, 51, 46, 48, 47, 48, 51, 48, 50]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:52.19010671Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1341,"":"downstairs","":"downstairs","session_id":"93402788-2fd3-4971-a5a9-5f7b4f31a615"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:52.19011739Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1341,"":"downstairs","":"downstairs","session_id":"93402788-2fd3-4971-a5a9-5f7b4f31a615"} {"msg":"[2]R flush_numbers[0..12]: [80, 79, 85, 81, 85, 79, 81, 80, 81, 85, 81, 82]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:52.190127629Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1341,"":"downstairs","":"downstairs","session_id":"93402788-2fd3-4971-a5a9-5f7b4f31a615"} {"msg":"[2]R generation[0..12]: [47, 46, 51, 48, 51, 46, 48, 47, 48, 51, 48, 50]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:52.190138069Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1341,"":"downstairs","":"downstairs","session_id":"93402788-2fd3-4971-a5a9-5f7b4f31a615"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:52.190149099Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1341,"":"downstairs","":"downstairs","session_id":"93402788-2fd3-4971-a5a9-5f7b4f31a615"} {"msg":"Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:52.190158428Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1341,"":"downstairs","":"downstairs","session_id":"93402788-2fd3-4971-a5a9-5f7b4f31a615"} {"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:52.190168008Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1341,"":"downstairs","":"downstairs","session_id":"93402788-2fd3-4971-a5a9-5f7b4f31a615"} {"msg":"Next flush: 86","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:52.190177238Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1341,"":"downstairs","":"downstairs","session_id":"93402788-2fd3-4971-a5a9-5f7b4f31a615"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:52.190186698Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1341,"":"downstairs","":"downstairs","session_id":"93402788-2fd3-4971-a5a9-5f7b4f31a615"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:52.190196397Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1341,"session_id":"93402788-2fd3-4971-a5a9-5f7b4f31a615"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:52.190205097Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1341,"client":"0","":"downstairs","session_id":"93402788-2fd3-4971-a5a9-5f7b4f31a615"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:52.190214967Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1341,"client":"1","":"downstairs","session_id":"93402788-2fd3-4971-a5a9-5f7b4f31a615"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:52.190225936Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1341,"client":"2","":"downstairs","session_id":"93402788-2fd3-4971-a5a9-5f7b4f31a615"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:52.190235746Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1341,"session_id":"93402788-2fd3-4971-a5a9-5f7b4f31a615"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:52.190244786Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1341,"session_id":"93402788-2fd3-4971-a5a9-5f7b4f31a615"} {"msg":"db20afd4-c9f5-43f3-b81a-b0f956ac0242 is now active with session: 93402788-2fd3-4971-a5a9-5f7b4f31a615","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:52.190254706Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1341,"session_id":"93402788-2fd3-4971-a5a9-5f7b4f31a615"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:52.190263335Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1341,"session_id":"93402788-2fd3-4971-a5a9-5f7b4f31a615"} {"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:52.190272315Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1341} {"msg":"Activated sub_volume db20afd4-c9f5-43f3-b81a-b0f956ac0242","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:52.190280325Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1341} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:52.190288435Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1341,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:52.190625605Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1341,"task":"crutest"} 01/30 Write block 195 len 16384 data: 3 3 3 3 02/30 Write block 442 len 28672 data: 4 3 2 2 2 2 2 03/30 Read block 186 len 4096 04/30 Write block 418 len 20480 data: 2 3 3 3 3 05/30 Write block 199 len 4096 data: 3 06/30 Write block 453 len 8192 data: 2 2 07/30 Read block 124 len 4096 08/30 Write block 307 len 36864 data: 4 4 4 4 4 5 5 5 5 09/30 Read block 487 len 36864 10/30 Write block 463 len 4096 data: 2 11/30 Read block 369 len 8192 12/30 Write block 456 len 8192 data: 3 4 13/30 Write block 297 len 20480 data: 9 7 6 6 5 14/30 Read block 167 len 40960 15/30 Read block 303 len 16384 16/30 Read block 276 len 40960 17/30 Flush 18/30 Write block 532 len 36864 data: 8 6 7 10 8 8 8 8 8 19/30 Write block 113 len 4096 data: 5 20/30 Read block 9 len 8192 21/30 Read block 550 len 16384 22/30 Read block 365 len 8192 23/30 Write block 449 len 20480 data: 2 3 3 2 3 24/30 Read block 547 len 24576 25/30 Flush 26/30 Write block 255 len 24576 data: 6 5 3 3 3 3 27/30 Read block 550 len 8192 28/30 Read block 55 len 40960 29/30 Read block 331 len 8192 30/30 Read block 125 len 24576 ---------------------------------------------------------------- Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Sent Done Done false 1001 Acked Write 7 Sent Done Done false 1002 Acked Read 1 Sent Done Done false 1003 Acked Write 5 Sent Done Done false 1004 Acked Write 1 Sent Done Done false 1005 Acked Write 2 Sent Done Done false 1006 Acked Read 1 Sent Done Done false 1007 Acked Write 9 Sent Done Done false 1008 Acked Read 9 Sent Done Done false 1009 Acked Write 1 Sent Done Done false 1010 Acked Read 2 Sent Done Done false 1011 Acked Write 2 Sent Done Done false 1012 Acked Write 5 Sent Done Done false 1013 Acked Read 10 Sent Done Done false 1014 Acked Read 4 Sent Done Done false 1015 Acked Read 10 Sent Done Done false 1016 Acked Flush 0 Sent Done Done false 1017 Acked Write 9 Sent Done Done false 1018 Acked Write 1 Sent Done Done false 1019 Acked Read 2 Sent Done Done false 1020 Acked Read 4 Sent Done Done false 1021 Acked Read 2 Sent Done Done false 1022 Acked Write 5 Sent Done Done false 1023 Acked Read 6 Sent Done Done false 1024 Acked Flush 0 Sent Done Done false 1025 Acked Write 6 Sent Done Done false 1026 Acked Read 2 Sent Done Done false 1027 Acked Read 10 Sent Done Done false 1028 Acked Read 2 Sent Done Done false 1029 Acked Read 6 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 0 0 30 Done 0 30 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:43:52.199006133Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1341} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:52.199192428Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1341,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 47 47 47 80 80 80 F F F 1 020-039 46 46 46 79 79 79 F F F 2 040-059 51 51 51 85 85 85 F F F 3 060-079 48 48 48 81 81 81 F F F 4 080-099 51 51 51 85 85 85 F F F 5 100-119 46 52 52 79 87 87 F F F <--- 6 120-139 48 48 48 81 81 81 F F F 7 140-159 47 47 47 80 80 80 F F F 8 160-179 48 48 48 81 81 81 F F F 9 180-199 51 52 52 85 86 86 F F F <--- 10 200-219 48 48 48 81 81 81 F F F 11 220-239 50 50 50 82 82 82 F F F 12 240-259 45 45 45 78 78 78 F T T <--- 13 260-279 47 47 47 80 80 80 F T T <--- 14 280-299 50 52 52 84 86 86 F F F <--- 15 300-319 48 52 52 81 86 86 F F F <--- 16 320-339 50 50 50 84 84 84 F F F 17 340-359 48 48 48 81 81 81 F F F 18 360-379 48 48 48 81 81 81 F F F 19 380-399 51 51 51 85 85 85 F F F 20 400-419 42 52 52 73 86 86 F F F <--- 21 420-439 36 52 52 63 86 86 F F F <--- 22 440-459 42 52 52 73 87 87 F F F <--- 23 460-479 51 52 52 85 86 86 F F F <--- 24 480-499 51 51 51 85 85 85 F F F 25 500-519 39 39 39 69 69 69 F F F 26 520-539 46 52 52 79 87 87 F F F <--- 27 540-559 48 52 52 81 87 87 F F F <--- 28 560-579 50 50 50 84 84 84 F F F 29 580-599 48 48 48 81 81 81 F F F Max gen: 52, Max flush: 87 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 26 Verifying data now No change after verify Loop: 26 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 47 47 47 80 80 80 F F F 1 020-039 46 46 46 79 79 79 F F F 2 040-059 51 51 51 85 85 85 F F F 3 060-079 48 48 48 81 81 81 F F F 4 080-099 51 51 51 85 85 85 F F F 5 100-119 52 52 52 87 87 87 F F F 6 120-139 48 48 48 81 81 81 F F F 7 140-159 47 47 47 80 80 80 F F F 8 160-179 48 48 48 81 81 81 F F F 9 180-199 52 52 52 86 86 86 F F F 10 200-219 48 48 48 81 81 81 F F F 11 220-239 50 50 50 82 82 82 F F F 12 240-259 53 53 53 88 88 88 F F F 13 260-279 53 53 53 88 88 88 F F F 14 280-299 52 52 52 86 86 86 F F F 15 300-319 52 52 52 86 86 86 F F F 16 320-339 50 50 50 84 84 84 F F F 17 340-359 48 48 48 81 81 81 F F F 18 360-379 48 48 48 81 81 81 F F F 19 380-399 51 51 51 85 85 85 F F F 20 400-419 52 52 52 86 86 86 F F F 21 420-439 52 52 52 86 86 86 F F F 22 440-459 52 52 52 87 87 87 F F F 23 460-479 52 52 52 86 86 86 F F F 24 480-499 51 51 51 85 85 85 F F F 25 500-519 39 39 39 69 69 69 F F F 26 520-539 52 52 52 87 87 87 F F F 27 540-559 52 52 52 87 87 87 F F F 28 560-579 50 50 50 84 84 84 F F F 29 580-599 48 48 48 81 81 81 F F F Max gen: 53, Max flush: 88 Begin loop 27 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:56.441571958Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1349,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:43:56.448643734Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1349,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:56.448699892Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1349,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:56.455333981Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1349,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:56.455596523Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1349} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:56.455623073Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1349} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:56.455631522Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1349} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:56.455642992Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1349} {"msg":"Crucible c41b76cb-2806-485b-b238-af75672a4b1c has session id: ad555119-5f85-4708-920e-c80759782518","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:56.456005502Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1349,"session_id":"ad555119-5f85-4708-920e-c80759782518"} {"msg":"Upstairs opts: Upstairs UUID: c41b76cb-2806-485b-b238-af75672a4b1c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:56.456034761Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1349,"session_id":"ad555119-5f85-4708-920e-c80759782518"} {"msg":"Crucible stats registered with UUID: c41b76cb-2806-485b-b238-af75672a4b1c","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:56.45604566Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1349,"session_id":"ad555119-5f85-4708-920e-c80759782518"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:43:56.45605507Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1349,"session_id":"ad555119-5f85-4708-920e-c80759782518"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:56.456186786Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1349,"":"io task","client":"0","":"downstairs","session_id":"ad555119-5f85-4708-920e-c80759782518"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:56.456220115Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1349,"":"io task","client":"2","":"downstairs","session_id":"ad555119-5f85-4708-920e-c80759782518"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:56.456231035Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1349,"":"io task","client":"1","":"downstairs","session_id":"ad555119-5f85-4708-920e-c80759782518"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:56.456348852Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1349,"task":"crutest"} {"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:56.456364521Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1349} {"msg":"c41b76cb-2806-485b-b238-af75672a4b1c active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:56.456381421Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1349,"session_id":"ad555119-5f85-4708-920e-c80759782518"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:56.45639017Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1349,"client":"0","":"downstairs","session_id":"ad555119-5f85-4708-920e-c80759782518"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:56.45640001Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1349,"client":"1","":"downstairs","session_id":"ad555119-5f85-4708-920e-c80759782518"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:56.45640956Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1349,"client":"2","":"downstairs","session_id":"ad555119-5f85-4708-920e-c80759782518"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:56.45641908Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1349,"":"io task","client":"2","":"downstairs","session_id":"ad555119-5f85-4708-920e-c80759782518"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:56.456441079Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1349,"":"io task","client":"0","":"downstairs","session_id":"ad555119-5f85-4708-920e-c80759782518"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:56.456467658Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1349,"":"io task","client":"1","":"downstairs","session_id":"ad555119-5f85-4708-920e-c80759782518"} {"msg":"ds_connection connected from Ok(127.0.0.1:40180)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:56.456551536Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1349,"":"io task","client":"2","":"downstairs","session_id":"ad555119-5f85-4708-920e-c80759782518"} {"msg":"ds_connection connected from Ok(127.0.0.1:64921)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:56.456576905Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1349,"":"io task","client":"1","":"downstairs","session_id":"ad555119-5f85-4708-920e-c80759782518"} {"msg":"ds_connection connected from Ok(127.0.0.1:50478)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:56.456596055Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1349,"":"io task","client":"0","":"downstairs","session_id":"ad555119-5f85-4708-920e-c80759782518"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:56.457174748Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1349,"client":"2","":"downstairs","session_id":"ad555119-5f85-4708-920e-c80759782518"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:56.457197887Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1349,"client":"1","":"downstairs","session_id":"ad555119-5f85-4708-920e-c80759782518"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:56.457322244Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1349,"client":"0","":"downstairs","session_id":"ad555119-5f85-4708-920e-c80759782518"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:56.45745976Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1349,"session_id":"ad555119-5f85-4708-920e-c80759782518"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:56.457502638Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1349,"session_id":"ad555119-5f85-4708-920e-c80759782518"} {"msg":"[0]R flush_numbers[0..12]: [80, 79, 85, 81, 85, 87, 81, 80, 81, 86, 81, 82]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:56.457590136Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1349,"":"downstairs","":"downstairs","session_id":"ad555119-5f85-4708-920e-c80759782518"} {"msg":"[0]R generation[0..12]: [47, 46, 51, 48, 51, 52, 48, 47, 48, 52, 48, 50]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:56.457601146Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1349,"":"downstairs","":"downstairs","session_id":"ad555119-5f85-4708-920e-c80759782518"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:56.457612015Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1349,"":"downstairs","":"downstairs","session_id":"ad555119-5f85-4708-920e-c80759782518"} {"msg":"[1]R flush_numbers[0..12]: [80, 79, 85, 81, 85, 87, 81, 80, 81, 86, 81, 82]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:56.457622465Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1349,"":"downstairs","":"downstairs","session_id":"ad555119-5f85-4708-920e-c80759782518"} {"msg":"[1]R generation[0..12]: [47, 46, 51, 48, 51, 52, 48, 47, 48, 52, 48, 50]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:56.457632795Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1349,"":"downstairs","":"downstairs","session_id":"ad555119-5f85-4708-920e-c80759782518"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:56.457646064Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1349,"":"downstairs","":"downstairs","session_id":"ad555119-5f85-4708-920e-c80759782518"} {"msg":"[2]R flush_numbers[0..12]: [80, 79, 85, 81, 85, 87, 81, 80, 81, 86, 81, 82]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:56.457656494Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1349,"":"downstairs","":"downstairs","session_id":"ad555119-5f85-4708-920e-c80759782518"} {"msg":"[2]R generation[0..12]: [47, 46, 51, 48, 51, 52, 48, 47, 48, 52, 48, 50]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:56.457666814Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1349,"":"downstairs","":"downstairs","session_id":"ad555119-5f85-4708-920e-c80759782518"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:56.457677593Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1349,"":"downstairs","":"downstairs","session_id":"ad555119-5f85-4708-920e-c80759782518"} {"msg":"Max found gen is 54","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:56.457687003Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1349,"":"downstairs","":"downstairs","session_id":"ad555119-5f85-4708-920e-c80759782518"} {"msg":"Generation requested: 54 >= found:54","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:56.457696593Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1349,"":"downstairs","":"downstairs","session_id":"ad555119-5f85-4708-920e-c80759782518"} {"msg":"Next flush: 89","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:56.457705773Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1349,"":"downstairs","":"downstairs","session_id":"ad555119-5f85-4708-920e-c80759782518"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:56.457715002Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1349,"":"downstairs","":"downstairs","session_id":"ad555119-5f85-4708-920e-c80759782518"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:56.457724582Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1349,"session_id":"ad555119-5f85-4708-920e-c80759782518"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:56.457733192Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1349,"client":"0","":"downstairs","session_id":"ad555119-5f85-4708-920e-c80759782518"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:56.457742791Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1349,"client":"1","":"downstairs","session_id":"ad555119-5f85-4708-920e-c80759782518"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:56.457752351Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1349,"client":"2","":"downstairs","session_id":"ad555119-5f85-4708-920e-c80759782518"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:56.457762121Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1349,"session_id":"ad555119-5f85-4708-920e-c80759782518"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:56.457771071Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1349,"session_id":"ad555119-5f85-4708-920e-c80759782518"} {"msg":"c41b76cb-2806-485b-b238-af75672a4b1c is now active with session: ad555119-5f85-4708-920e-c80759782518","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:56.45778098Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1349,"session_id":"ad555119-5f85-4708-920e-c80759782518"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:56.457791Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1349,"session_id":"ad555119-5f85-4708-920e-c80759782518"} {"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:56.45780034Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1349} {"msg":"Activated sub_volume c41b76cb-2806-485b-b238-af75672a4b1c","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:56.45780841Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1349} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:56.457816539Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1349,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:56.458120391Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1349,"task":"crutest"} 01/30 Write block 94 len 24576 data: 8 8 8 6 5 5 02/30 Read block 446 len 8192 03/30 Read block 380 len 36864 04/30 Read block 296 len 8192 05/30 Write block 229 len 24576 data: 7 6 6 5 5 5 06/30 Read block 201 len 32768 07/30 Read block 566 len 28672 08/30 Write block 165 len 12288 data: 6 6 6 09/30 Write block 36 len 40960 data: 3 3 3 3 3 2 2 2 2 2 10/30 Write block 427 len 4096 data: 3 11/30 Read block 486 len 16384 12/30 Flush 13/30 Write block 340 len 24576 data: 5 5 4 5 6 6 14/30 Read block 375 len 4096 15/30 Read block 545 len 40960 16/30 Read block 570 len 20480 17/30 Read block 422 len 28672 18/30 Read block 110 len 8192 19/30 Write block 361 len 32768 data: 5 6 7 9 9 7 7 6 20/30 Read block 99 len 8192 21/30 Flush 22/30 Write block 116 len 4096 data: 5 23/30 Read block 305 len 24576 24/30 Read block 508 len 32768 25/30 Read block 481 len 12288 26/30 Write block 65 len 24576 data: 4 4 3 2 2 2 27/30 Read block 588 len 16384 28/30 Write block 51 len 32768 data: 3 4 4 5 5 4 4 5 29/30 Write block 301 len 20480 data: 6 3 3 4 4 30/30 Read block 50 len 40960 ---------------------------------------------------------------- Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Sent Done Done false 1001 Acked Read 2 Sent Done Done false 1002 Acked Read 9 Sent Done Done false 1003 Acked Read 2 Sent Done Done false 1004 Acked Write 6 Sent Done Done false 1005 Acked Read 8 Sent Done Done false 1006 Acked Read 7 Sent Done Done false 1007 Acked Write 3 Sent Done Done false 1008 Acked Write 10 Sent Done Done false 1009 Acked Write 1 Sent Done Done false 1010 Acked Read 4 Sent Done Done false 1011 Acked Flush 0 Sent Done Done false 1012 Acked Write 6 Sent Done Done false 1013 Acked Read 1 Sent Done Done false 1014 Acked Read 10 Sent Done Done false 1015 Acked Read 5 Sent Done Done false 1016 Acked Read 7 Sent Done Done false 1017 Acked Read 2 Sent Done Done false 1018 Acked Write 8 Sent Done Done false 1019 Acked Read 2 Sent Done Done false 1020 Acked Flush 0 Sent Done Done false 1021 Acked Write 1 Sent Done Done false 1022 Acked Read 6 Sent Done Done false 1023 Acked Read 8 Sent Done Done false 1024 Acked Read 3 Sent Done Done false 1025 Acked Write 6 Sent Done Done false 1026 Acked Read 4 Sent Done Done false 1027 Acked Write 8 Sent Done Done false 1028 Acked Write 5 Sent Done Done false 1029 Acked Read 10 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 1 0 31 Done 0 29 30 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:43:56.467527139Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1349} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:43:56.467703924Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1349,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 47 47 47 80 80 80 F F F 1 020-039 46 54 54 79 89 89 F F F <--- 2 040-059 51 54 54 85 89 89 F T T <--- 3 060-079 48 48 48 81 81 81 F T T <--- 4 080-099 51 54 54 85 89 89 F F F <--- 5 100-119 52 52 52 87 87 87 F T T <--- 6 120-139 48 48 48 81 81 81 F F F 7 140-159 47 47 47 80 80 80 F F F 8 160-179 48 54 54 81 89 89 F F F <--- 9 180-199 52 52 52 86 86 86 F F F 10 200-219 48 48 48 81 81 81 F F F 11 220-239 50 54 54 82 89 89 F F F <--- 12 240-259 53 53 53 88 88 88 F F F 13 260-279 53 53 53 88 88 88 F F F 14 280-299 52 52 52 86 86 86 F F F 15 300-319 52 52 52 86 86 86 F T T <--- 16 320-339 50 50 50 84 84 84 F F F 17 340-359 48 54 54 81 90 90 F F F <--- 18 360-379 48 54 54 81 90 90 F F F <--- 19 380-399 51 51 51 85 85 85 F F F 20 400-419 52 52 52 86 86 86 F F F 21 420-439 52 54 54 86 89 89 F F F <--- 22 440-459 52 52 52 87 87 87 F F F 23 460-479 52 52 52 86 86 86 F F F 24 480-499 51 51 51 85 85 85 F F F 25 500-519 39 39 39 69 69 69 F F F 26 520-539 52 52 52 87 87 87 F F F 27 540-559 52 52 52 87 87 87 F F F 28 560-579 50 50 50 84 84 84 F F F 29 580-599 48 48 48 81 81 81 F F F Max gen: 54, Max flush: 90 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 27 Verifying data now No change after verify Loop: 27 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 47 47 47 80 80 80 F F F 1 020-039 54 54 54 89 89 89 F F F 2 040-059 55 55 55 91 91 91 F F F 3 060-079 55 55 55 91 91 91 F F F 4 080-099 54 54 54 89 89 89 F F F 5 100-119 55 55 55 91 91 91 F F F 6 120-139 48 48 48 81 81 81 F F F 7 140-159 47 47 47 80 80 80 F F F 8 160-179 54 54 54 89 89 89 F F F 9 180-199 52 52 52 86 86 86 F F F 10 200-219 48 48 48 81 81 81 F F F 11 220-239 54 54 54 89 89 89 F F F 12 240-259 53 53 53 88 88 88 F F F 13 260-279 53 53 53 88 88 88 F F F 14 280-299 52 52 52 86 86 86 F F F 15 300-319 55 55 55 91 91 91 F F F 16 320-339 50 50 50 84 84 84 F F F 17 340-359 54 54 54 90 90 90 F F F 18 360-379 54 54 54 90 90 90 F F F 19 380-399 51 51 51 85 85 85 F F F 20 400-419 52 52 52 86 86 86 F F F 21 420-439 54 54 54 89 89 89 F F F 22 440-459 52 52 52 87 87 87 F F F 23 460-479 52 52 52 86 86 86 F F F 24 480-499 51 51 51 85 85 85 F F F 25 500-519 39 39 39 69 69 69 F F F 26 520-539 52 52 52 87 87 87 F F F 27 540-559 52 52 52 87 87 87 F F F 28 560-579 50 50 50 84 84 84 F F F 29 580-599 48 48 48 81 81 81 F F F Max gen: 55, Max flush: 91 Begin loop 28 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:00.715350192Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1357,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:00.723525877Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1357,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:00.723801229Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1357} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:00.723828668Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1357} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:00.723837588Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1357} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:00.723849047Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1357} {"msg":"Crucible 51e3d31c-ef22-4cd8-9e60-e2de0dbe0705 has session id: 78cb134f-3714-4a6e-998e-e2b531df0f71","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:00.724176108Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1357,"session_id":"78cb134f-3714-4a6e-998e-e2b531df0f71"} {"msg":"Upstairs opts: Upstairs UUID: 51e3d31c-ef22-4cd8-9e60-e2de0dbe0705, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:00.724202227Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1357,"session_id":"78cb134f-3714-4a6e-998e-e2b531df0f71"} {"msg":"Crucible stats registered with UUID: 51e3d31c-ef22-4cd8-9e60-e2de0dbe0705","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:00.724212277Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1357,"session_id":"78cb134f-3714-4a6e-998e-e2b531df0f71"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:44:00.724221316Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1357,"session_id":"78cb134f-3714-4a6e-998e-e2b531df0f71"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:00.724372572Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1357,"":"io task","client":"1","":"downstairs","session_id":"78cb134f-3714-4a6e-998e-e2b531df0f71"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:00.724396321Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1357,"":"io task","client":"0","":"downstairs","session_id":"78cb134f-3714-4a6e-998e-e2b531df0f71"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:00.724406571Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1357,"":"io task","client":"2","":"downstairs","session_id":"78cb134f-3714-4a6e-998e-e2b531df0f71"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:00.724522858Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1357,"task":"crutest"} {"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:00.724556927Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1357} {"msg":"51e3d31c-ef22-4cd8-9e60-e2de0dbe0705 active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:00.724565727Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1357,"session_id":"78cb134f-3714-4a6e-998e-e2b531df0f71"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:00.724574516Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1357,"client":"0","":"downstairs","session_id":"78cb134f-3714-4a6e-998e-e2b531df0f71"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:00.724584266Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1357,"client":"1","":"downstairs","session_id":"78cb134f-3714-4a6e-998e-e2b531df0f71"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:00.724593826Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1357,"client":"2","":"downstairs","session_id":"78cb134f-3714-4a6e-998e-e2b531df0f71"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:00.724603345Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1357,"":"io task","client":"2","":"downstairs","session_id":"78cb134f-3714-4a6e-998e-e2b531df0f71"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:00.724618155Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1357,"":"io task","client":"0","":"downstairs","session_id":"78cb134f-3714-4a6e-998e-e2b531df0f71"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:00.724628125Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1357,"":"io task","client":"1","":"downstairs","session_id":"78cb134f-3714-4a6e-998e-e2b531df0f71"} {"msg":"ds_connection connected from Ok(127.0.0.1:62170)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:00.724647864Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1357,"":"io task","client":"0","":"downstairs","session_id":"78cb134f-3714-4a6e-998e-e2b531df0f71"} {"msg":"ds_connection connected from Ok(127.0.0.1:41772)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:00.724731692Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1357,"":"io task","client":"1","":"downstairs","session_id":"78cb134f-3714-4a6e-998e-e2b531df0f71"} {"msg":"ds_connection connected from Ok(127.0.0.1:63183)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:00.72479533Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1357,"":"io task","client":"2","":"downstairs","session_id":"78cb134f-3714-4a6e-998e-e2b531df0f71"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:00.725331074Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1357,"client":"0","":"downstairs","session_id":"78cb134f-3714-4a6e-998e-e2b531df0f71"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:00.725363714Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1357,"client":"1","":"downstairs","session_id":"78cb134f-3714-4a6e-998e-e2b531df0f71"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:00.725387583Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1357,"client":"2","":"downstairs","session_id":"78cb134f-3714-4a6e-998e-e2b531df0f71"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:00.725645765Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1357,"session_id":"78cb134f-3714-4a6e-998e-e2b531df0f71"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:00.725678054Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1357,"session_id":"78cb134f-3714-4a6e-998e-e2b531df0f71"} {"msg":"[0]R flush_numbers[0..12]: [80, 89, 91, 91, 89, 91, 81, 80, 89, 86, 81, 89]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:00.725724323Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1357,"":"downstairs","":"downstairs","session_id":"78cb134f-3714-4a6e-998e-e2b531df0f71"} {"msg":"[0]R generation[0..12]: [47, 54, 55, 55, 54, 55, 48, 47, 54, 52, 48, 54]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:00.725735133Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1357,"":"downstairs","":"downstairs","session_id":"78cb134f-3714-4a6e-998e-e2b531df0f71"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:00.725746063Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1357,"":"downstairs","":"downstairs","session_id":"78cb134f-3714-4a6e-998e-e2b531df0f71"} {"msg":"[1]R flush_numbers[0..12]: [80, 89, 91, 91, 89, 91, 81, 80, 89, 86, 81, 89]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:00.725756362Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1357,"":"downstairs","":"downstairs","session_id":"78cb134f-3714-4a6e-998e-e2b531df0f71"} {"msg":"[1]R generation[0..12]: [47, 54, 55, 55, 54, 55, 48, 47, 54, 52, 48, 54]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:00.725766592Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1357,"":"downstairs","":"downstairs","session_id":"78cb134f-3714-4a6e-998e-e2b531df0f71"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:00.725777362Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1357,"":"downstairs","":"downstairs","session_id":"78cb134f-3714-4a6e-998e-e2b531df0f71"} {"msg":"[2]R flush_numbers[0..12]: [80, 89, 91, 91, 89, 91, 81, 80, 89, 86, 81, 89]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:00.725787641Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1357,"":"downstairs","":"downstairs","session_id":"78cb134f-3714-4a6e-998e-e2b531df0f71"} {"msg":"[2]R generation[0..12]: [47, 54, 55, 55, 54, 55, 48, 47, 54, 52, 48, 54]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:00.725797891Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1357,"":"downstairs","":"downstairs","session_id":"78cb134f-3714-4a6e-998e-e2b531df0f71"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:00.725808661Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1357,"":"downstairs","":"downstairs","session_id":"78cb134f-3714-4a6e-998e-e2b531df0f71"} {"msg":"Max found gen is 56","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:00.72581795Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1357,"":"downstairs","":"downstairs","session_id":"78cb134f-3714-4a6e-998e-e2b531df0f71"} {"msg":"Generation requested: 56 >= found:56","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:00.72582784Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1357,"":"downstairs","":"downstairs","session_id":"78cb134f-3714-4a6e-998e-e2b531df0f71"} {"msg":"Next flush: 92","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:00.72583877Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1357,"":"downstairs","":"downstairs","session_id":"78cb134f-3714-4a6e-998e-e2b531df0f71"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:00.72584808Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1357,"":"downstairs","":"downstairs","session_id":"78cb134f-3714-4a6e-998e-e2b531df0f71"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:00.725857719Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1357,"session_id":"78cb134f-3714-4a6e-998e-e2b531df0f71"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:00.725866349Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1357,"client":"0","":"downstairs","session_id":"78cb134f-3714-4a6e-998e-e2b531df0f71"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:00.725876159Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1357,"client":"1","":"downstairs","session_id":"78cb134f-3714-4a6e-998e-e2b531df0f71"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:00.725885848Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1357,"client":"2","":"downstairs","session_id":"78cb134f-3714-4a6e-998e-e2b531df0f71"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:00.725895788Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1357,"session_id":"78cb134f-3714-4a6e-998e-e2b531df0f71"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:00.725904968Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1357,"session_id":"78cb134f-3714-4a6e-998e-e2b531df0f71"} {"msg":"51e3d31c-ef22-4cd8-9e60-e2de0dbe0705 is now active with session: 78cb134f-3714-4a6e-998e-e2b531df0f71","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:00.725914798Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1357,"session_id":"78cb134f-3714-4a6e-998e-e2b531df0f71"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:00.725923407Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1357,"session_id":"78cb134f-3714-4a6e-998e-e2b531df0f71"} {"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:00.725932427Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1357} {"msg":"Activated sub_volume 51e3d31c-ef22-4cd8-9e60-e2de0dbe0705","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:00.725940327Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1357} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:00.725948367Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1357,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:00.726235148Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1357,"task":"crutest"} 01/30 Write block 558 len 40960 data: 9 10 7 6 7 5 6 6 6 3 02/30 Write block 159 len 8192 data: 4 5 03/30 Read block 126 len 16384 04/30 Read block 323 len 12288 05/30 Read block 235 len 28672 06/30 Read block 579 len 28672 07/30 Write block 96 len 32768 data: 9 7 6 6 7 3 3 3 08/30 Read block 13 len 16384 09/30 Flush 10/30 Write block 481 len 4096 data: 8 11/30 Write block 139 len 40960 data: 7 6 9 8 7 7 7 8 6 5 12/30 Flush 13/30 Write block 490 len 20480 data: 4 4 3 3 2 14/30 Write block 78 len 32768 data: 6 5 5 4 4 5 5 4 15/30 Read block 11 len 16384 16/30 Write block 137 len 20480 data: 6 6 8 7 10 17/30 Read block 511 len 36864 18/30 Write block 67 len 36864 data: 4 3 3 3 3 3 3 3 4 19/30 Write block 324 len 32768 data: 6 6 5 5 4 4 5 5 20/30 Write block 214 len 4096 data: 7 21/30 Write block 525 len 12288 data: 6 5 5 22/30 Read block 335 len 16384 23/30 Read block 561 len 40960 24/30 Write block 130 len 12288 data: 5 7 7 25/30 Read block 123 len 32768 26/30 Write block 570 len 36864 data: 4 4 3 4 4 4 6 6 6 27/30 Write block 189 len 4096 data: 3 28/30 Read block 490 len 32768 29/30 Write block 198 len 20480 data: 4 4 2 7 8 30/30 Write block 136 len 20480 data: 9 7 7 9 8 ---------------------------------------------------------------- Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:21 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1009 Acked Write 1 Done Done Sent false 1010 Acked Write 10 Done Done Sent false 1011 Acked Flush 0 Done Done Sent false 1012 Acked Write 5 Done Done Sent false 1013 Acked Write 8 Done Done Sent false 1014 Acked Read 4 Done Done Sent false 1015 Acked Write 5 Done Done Sent false 1016 Acked Read 9 Done Done Sent false 1017 Acked Write 9 Done Done Sent false 1018 Acked Write 8 Done Done Sent false 1019 Acked Write 1 Done Done Sent false 1020 Acked Write 3 Done Done Sent false 1021 Acked Read 4 Done Done Sent false 1022 Acked Read 10 Done Done Sent false 1023 Acked Write 3 Done Done Sent false 1024 Acked Read 8 Done Done Sent false 1025 Acked Write 9 Done Done Sent false 1026 Acked Write 1 Done Done Sent false 1027 Acked Read 8 Done Done Sent false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 21 25 Done 19 19 0 38 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1011)) Some(JobId(1011)) Some(JobId(1008)) Downstairs last five completed: 1008 1007 1006 1005 1004 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:44:00.734753973Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1357} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:00.734947317Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1357,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 47 47 47 80 80 80 F F F 1 020-039 54 54 54 89 89 89 F F F 2 040-059 55 55 55 91 91 91 F F F 3 060-079 55 55 55 91 91 91 T T F <--- 4 080-099 56 56 56 92 92 92 T T F <--- 5 100-119 56 56 56 92 92 92 F F F 6 120-139 56 56 48 93 93 81 T T F <--- 7 140-159 56 56 56 93 93 92 T T F <--- 8 160-179 56 56 56 92 92 92 F F F 9 180-199 52 52 52 86 86 86 T T F <--- 10 200-219 48 48 48 81 81 81 T T F <--- 11 220-239 54 54 54 89 89 89 F F F 12 240-259 53 53 53 88 88 88 F F F 13 260-279 53 53 53 88 88 88 F F F 14 280-299 52 52 52 86 86 86 F F F 15 300-319 55 55 55 91 91 91 F F F 16 320-339 50 50 50 84 84 84 T T F <--- 17 340-359 54 54 54 90 90 90 F F F 18 360-379 54 54 54 90 90 90 F F F 19 380-399 51 51 51 85 85 85 F F F 20 400-419 52 52 52 86 86 86 F F F 21 420-439 54 54 54 89 89 89 F F F 22 440-459 52 52 52 87 87 87 F F F 23 460-479 52 52 52 86 86 86 F F F 24 480-499 56 56 51 93 93 85 T T F <--- 25 500-519 39 39 39 69 69 69 F F F 26 520-539 52 52 52 87 87 87 T T F <--- 27 540-559 56 56 56 92 92 92 F F F 28 560-579 56 56 56 92 92 92 T T F <--- 29 580-599 48 48 48 81 81 81 F F F Max gen: 56, Max flush: 93 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 28 Verifying data now No change after verify Loop: 28 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 47 47 47 80 80 80 F F F 1 020-039 54 54 54 89 89 89 F F F 2 040-059 55 55 55 91 91 91 F F F 3 060-079 57 57 57 94 94 94 F F F 4 080-099 57 57 57 94 94 94 F F F 5 100-119 56 56 56 92 92 92 F F F 6 120-139 57 57 57 94 94 94 F F F 7 140-159 57 57 57 94 94 94 F F F 8 160-179 56 56 56 92 92 92 F F F 9 180-199 57 57 57 94 94 94 F F F 10 200-219 57 57 57 94 94 94 F F F 11 220-239 54 54 54 89 89 89 F F F 12 240-259 53 53 53 88 88 88 F F F 13 260-279 53 53 53 88 88 88 F F F 14 280-299 52 52 52 86 86 86 F F F 15 300-319 55 55 55 91 91 91 F F F 16 320-339 57 57 57 94 94 94 F F F 17 340-359 54 54 54 90 90 90 F F F 18 360-379 54 54 54 90 90 90 F F F 19 380-399 51 51 51 85 85 85 F F F 20 400-419 52 52 52 86 86 86 F F F 21 420-439 54 54 54 89 89 89 F F F 22 440-459 52 52 52 87 87 87 F F F 23 460-479 52 52 52 86 86 86 F F F 24 480-499 57 57 57 94 94 94 F F F 25 500-519 39 39 39 69 69 69 F F F 26 520-539 57 57 57 94 94 94 F F F 27 540-559 56 56 56 92 92 92 F F F 28 560-579 57 57 57 94 94 94 F F F 29 580-599 48 48 48 81 81 81 F F F Max gen: 57, Max flush: 94 Begin loop 29 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:04.974986734Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1365,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:04.983019773Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1365,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:04.983283315Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1365} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:04.983309764Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1365} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:04.983321224Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1365} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:04.983332874Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1365} {"msg":"Crucible 3b5df34d-57c9-43f1-bf32-79032f24a258 has session id: 31e370c6-24d6-4669-8847-7809f6ac4014","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:04.983649304Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1365,"session_id":"31e370c6-24d6-4669-8847-7809f6ac4014"} {"msg":"Upstairs opts: Upstairs UUID: 3b5df34d-57c9-43f1-bf32-79032f24a258, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:04.983679244Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1365,"session_id":"31e370c6-24d6-4669-8847-7809f6ac4014"} {"msg":"Crucible stats registered with UUID: 3b5df34d-57c9-43f1-bf32-79032f24a258","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:04.983689923Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1365,"session_id":"31e370c6-24d6-4669-8847-7809f6ac4014"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:44:04.983699103Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1365,"session_id":"31e370c6-24d6-4669-8847-7809f6ac4014"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:04.983844339Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1365,"":"io task","client":"0","":"downstairs","session_id":"31e370c6-24d6-4669-8847-7809f6ac4014"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:04.983873838Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1365,"":"io task","client":"1","":"downstairs","session_id":"31e370c6-24d6-4669-8847-7809f6ac4014"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:04.983884738Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1365,"":"io task","client":"2","":"downstairs","session_id":"31e370c6-24d6-4669-8847-7809f6ac4014"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:04.984009484Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1365,"task":"crutest"} {"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:04.984028103Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1365} {"msg":"3b5df34d-57c9-43f1-bf32-79032f24a258 active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:04.984040703Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1365,"session_id":"31e370c6-24d6-4669-8847-7809f6ac4014"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:04.984049503Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1365,"client":"0","":"downstairs","session_id":"31e370c6-24d6-4669-8847-7809f6ac4014"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:04.984059143Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1365,"client":"1","":"downstairs","session_id":"31e370c6-24d6-4669-8847-7809f6ac4014"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:04.984068672Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1365,"client":"2","":"downstairs","session_id":"31e370c6-24d6-4669-8847-7809f6ac4014"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:04.984080362Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1365,"":"io task","client":"2","":"downstairs","session_id":"31e370c6-24d6-4669-8847-7809f6ac4014"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:04.984090292Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1365,"":"io task","client":"0","":"downstairs","session_id":"31e370c6-24d6-4669-8847-7809f6ac4014"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:04.984100201Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1365,"":"io task","client":"1","":"downstairs","session_id":"31e370c6-24d6-4669-8847-7809f6ac4014"} {"msg":"ds_connection connected from Ok(127.0.0.1:48004)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:04.984170609Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1365,"":"io task","client":"2","":"downstairs","session_id":"31e370c6-24d6-4669-8847-7809f6ac4014"} {"msg":"ds_connection connected from Ok(127.0.0.1:41708)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:04.984188349Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1365,"":"io task","client":"0","":"downstairs","session_id":"31e370c6-24d6-4669-8847-7809f6ac4014"} {"msg":"ds_connection connected from Ok(127.0.0.1:63478)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:04.984264037Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1365,"":"io task","client":"1","":"downstairs","session_id":"31e370c6-24d6-4669-8847-7809f6ac4014"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:04.98483645Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1365,"client":"2","":"downstairs","session_id":"31e370c6-24d6-4669-8847-7809f6ac4014"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:04.984895258Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1365,"client":"0","":"downstairs","session_id":"31e370c6-24d6-4669-8847-7809f6ac4014"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:04.984907618Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1365,"client":"1","":"downstairs","session_id":"31e370c6-24d6-4669-8847-7809f6ac4014"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:04.985133382Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1365,"session_id":"31e370c6-24d6-4669-8847-7809f6ac4014"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:04.985168521Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1365,"session_id":"31e370c6-24d6-4669-8847-7809f6ac4014"} {"msg":"[0]R flush_numbers[0..12]: [80, 89, 91, 94, 94, 92, 94, 94, 92, 94, 94, 89]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:04.985242728Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1365,"":"downstairs","":"downstairs","session_id":"31e370c6-24d6-4669-8847-7809f6ac4014"} {"msg":"[0]R generation[0..12]: [47, 54, 55, 57, 57, 56, 57, 57, 56, 57, 57, 54]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:04.985253718Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1365,"":"downstairs","":"downstairs","session_id":"31e370c6-24d6-4669-8847-7809f6ac4014"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:04.985264858Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1365,"":"downstairs","":"downstairs","session_id":"31e370c6-24d6-4669-8847-7809f6ac4014"} {"msg":"[1]R flush_numbers[0..12]: [80, 89, 91, 94, 94, 92, 94, 94, 92, 94, 94, 89]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:04.985275278Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1365,"":"downstairs","":"downstairs","session_id":"31e370c6-24d6-4669-8847-7809f6ac4014"} {"msg":"[1]R generation[0..12]: [47, 54, 55, 57, 57, 56, 57, 57, 56, 57, 57, 54]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:04.985287877Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1365,"":"downstairs","":"downstairs","session_id":"31e370c6-24d6-4669-8847-7809f6ac4014"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:04.985298737Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1365,"":"downstairs","":"downstairs","session_id":"31e370c6-24d6-4669-8847-7809f6ac4014"} {"msg":"[2]R flush_numbers[0..12]: [80, 89, 91, 94, 94, 92, 94, 94, 92, 94, 94, 89]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:04.985309007Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1365,"":"downstairs","":"downstairs","session_id":"31e370c6-24d6-4669-8847-7809f6ac4014"} {"msg":"[2]R generation[0..12]: [47, 54, 55, 57, 57, 56, 57, 57, 56, 57, 57, 54]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:04.985319196Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1365,"":"downstairs","":"downstairs","session_id":"31e370c6-24d6-4669-8847-7809f6ac4014"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:04.985329896Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1365,"":"downstairs","":"downstairs","session_id":"31e370c6-24d6-4669-8847-7809f6ac4014"} {"msg":"Max found gen is 58","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:04.985339236Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1365,"":"downstairs","":"downstairs","session_id":"31e370c6-24d6-4669-8847-7809f6ac4014"} {"msg":"Generation requested: 58 >= found:58","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:04.985348775Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1365,"":"downstairs","":"downstairs","session_id":"31e370c6-24d6-4669-8847-7809f6ac4014"} {"msg":"Next flush: 95","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:04.985358145Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1365,"":"downstairs","":"downstairs","session_id":"31e370c6-24d6-4669-8847-7809f6ac4014"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:04.985367445Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1365,"":"downstairs","":"downstairs","session_id":"31e370c6-24d6-4669-8847-7809f6ac4014"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:04.985377455Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1365,"session_id":"31e370c6-24d6-4669-8847-7809f6ac4014"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:04.985386324Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1365,"client":"0","":"downstairs","session_id":"31e370c6-24d6-4669-8847-7809f6ac4014"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:04.985396364Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1365,"client":"1","":"downstairs","session_id":"31e370c6-24d6-4669-8847-7809f6ac4014"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:04.985406174Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1365,"client":"2","":"downstairs","session_id":"31e370c6-24d6-4669-8847-7809f6ac4014"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:04.985416073Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1365,"session_id":"31e370c6-24d6-4669-8847-7809f6ac4014"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:04.985425213Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1365,"session_id":"31e370c6-24d6-4669-8847-7809f6ac4014"} {"msg":"3b5df34d-57c9-43f1-bf32-79032f24a258 is now active with session: 31e370c6-24d6-4669-8847-7809f6ac4014","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:04.985436473Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1365,"session_id":"31e370c6-24d6-4669-8847-7809f6ac4014"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:04.985445273Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1365,"session_id":"31e370c6-24d6-4669-8847-7809f6ac4014"} {"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:04.985454372Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1365} {"msg":"Activated sub_volume 3b5df34d-57c9-43f1-bf32-79032f24a258","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:04.985462242Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1365} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:04.985470252Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1365,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:04.985794173Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1365,"task":"crutest"} 01/30 Write block 31 len 32768 data: 5 5 5 6 5 4 4 4 02/30 Write block 168 len 8192 data: 7 8 03/30 Read block 242 len 24576 04/30 Read block 126 len 4096 05/30 Write block 55 len 12288 data: 6 5 5 06/30 Read block 0 len 24576 07/30 Write block 517 len 8192 data: 3 5 08/30 Read block 530 len 24576 09/30 Read block 175 len 36864 10/30 Read block 339 len 4096 11/30 Write block 491 len 4096 data: 5 12/30 Read block 378 len 24576 13/30 Read block 255 len 36864 14/30 Read block 122 len 40960 15/30 Read block 53 len 32768 16/30 Read block 258 len 16384 17/30 Read block 75 len 16384 18/30 Write block 248 len 12288 data: 6 5 6 19/30 Write block 440 len 4096 data: 6 20/30 Read block 286 len 12288 21/30 Read block 83 len 8192 22/30 Read block 350 len 8192 23/30 Write block 172 len 36864 data: 6 6 5 6 6 6 5 4 3 24/30 Flush 25/30 Write block 267 len 12288 data: 2 3 3 26/30 Read block 497 len 4096 27/30 Write block 309 len 4096 data: 5 28/30 Write block 395 len 8192 data: 3 3 29/30 Read block 311 len 16384 30/30 Write block 568 len 36864 data: 4 3 5 5 4 5 5 5 7 ---------------------------------------------------------------- Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 3 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Read 4 Done Sent Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 1 4 Done 5 4 5 14 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:44:04.993822761Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1365} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:04.993955777Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1365,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 47 47 47 80 80 80 F F F 1 020-039 58 58 58 95 95 95 F F F 2 040-059 58 58 58 95 95 95 F F F 3 060-079 57 57 57 94 94 94 F F F 4 080-099 57 57 57 94 94 94 F F F 5 100-119 56 56 56 92 92 92 F F F 6 120-139 57 57 57 94 94 94 F F F 7 140-159 57 57 57 94 94 94 F F F 8 160-179 58 58 58 95 95 95 F F F 9 180-199 58 58 58 95 95 95 F F F 10 200-219 57 57 57 94 94 94 F F F 11 220-239 54 54 54 89 89 89 F F F 12 240-259 58 58 58 95 95 95 F F F 13 260-279 53 53 53 88 88 88 T T T 14 280-299 52 52 52 86 86 86 F F F 15 300-319 55 55 55 91 91 91 T T T 16 320-339 57 57 57 94 94 94 F F F 17 340-359 54 54 54 90 90 90 F F F 18 360-379 54 54 54 90 90 90 F F F 19 380-399 51 51 51 85 85 85 T T T 20 400-419 52 52 52 86 86 86 F F F 21 420-439 54 54 54 89 89 89 F F F 22 440-459 58 58 58 95 95 95 F F F 23 460-479 52 52 52 86 86 86 F F F 24 480-499 58 58 58 95 95 95 F F F 25 500-519 58 58 58 95 95 95 F F F 26 520-539 57 57 57 94 94 94 F F F 27 540-559 56 56 56 92 92 92 F F F 28 560-579 57 57 57 94 94 94 T F T <--- 29 580-599 48 48 48 81 81 81 F F F Max gen: 58, Max flush: 95 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 29 Verifying data now No change after verify Loop: 29 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 47 47 47 80 80 80 F F F 1 020-039 58 58 58 95 95 95 F F F 2 040-059 58 58 58 95 95 95 F F F 3 060-079 57 57 57 94 94 94 F F F 4 080-099 57 57 57 94 94 94 F F F 5 100-119 56 56 56 92 92 92 F F F 6 120-139 57 57 57 94 94 94 F F F 7 140-159 57 57 57 94 94 94 F F F 8 160-179 58 58 58 95 95 95 F F F 9 180-199 58 58 58 95 95 95 F F F 10 200-219 57 57 57 94 94 94 F F F 11 220-239 54 54 54 89 89 89 F F F 12 240-259 58 58 58 95 95 95 F F F 13 260-279 59 59 59 96 96 96 F F F 14 280-299 52 52 52 86 86 86 F F F 15 300-319 59 59 59 96 96 96 F F F 16 320-339 57 57 57 94 94 94 F F F 17 340-359 54 54 54 90 90 90 F F F 18 360-379 54 54 54 90 90 90 F F F 19 380-399 59 59 59 96 96 96 F F F 20 400-419 52 52 52 86 86 86 F F F 21 420-439 54 54 54 89 89 89 F F F 22 440-459 58 58 58 95 95 95 F F F 23 460-479 52 52 52 86 86 86 F F F 24 480-499 58 58 58 95 95 95 F F F 25 500-519 58 58 58 95 95 95 F F F 26 520-539 57 57 57 94 94 94 F F F 27 540-559 56 56 56 92 92 92 F F F 28 560-579 59 59 59 96 96 96 F F F 29 580-599 48 48 48 81 81 81 F F F Max gen: 59, Max flush: 96 Begin loop 30 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:09.225108131Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1373,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:44:09.232119399Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1373,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:09.232173457Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1373,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:09.238970421Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1373,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:09.239228804Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1373} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:09.239252493Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1373} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:09.239260883Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1373} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:09.239272293Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1373} {"msg":"Crucible c243d458-51c1-48ad-b110-62cb4006c091 has session id: c618c25c-4af6-4008-b195-3adafc86bfd7","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:09.239620503Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1373,"session_id":"c618c25c-4af6-4008-b195-3adafc86bfd7"} {"msg":"Upstairs opts: Upstairs UUID: c243d458-51c1-48ad-b110-62cb4006c091, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:09.239649942Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1373,"session_id":"c618c25c-4af6-4008-b195-3adafc86bfd7"} {"msg":"Crucible stats registered with UUID: c243d458-51c1-48ad-b110-62cb4006c091","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:09.239660761Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1373,"session_id":"c618c25c-4af6-4008-b195-3adafc86bfd7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:44:09.239669891Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1373,"session_id":"c618c25c-4af6-4008-b195-3adafc86bfd7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:09.239804707Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1373,"":"io task","client":"1","":"downstairs","session_id":"c618c25c-4af6-4008-b195-3adafc86bfd7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:09.239822417Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1373,"":"io task","client":"0","":"downstairs","session_id":"c618c25c-4af6-4008-b195-3adafc86bfd7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:09.239832406Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1373,"":"io task","client":"2","":"downstairs","session_id":"c618c25c-4af6-4008-b195-3adafc86bfd7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:09.239980022Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1373,"task":"crutest"} {"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:09.239995502Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1373} {"msg":"c243d458-51c1-48ad-b110-62cb4006c091 active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:09.240031761Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1373,"session_id":"c618c25c-4af6-4008-b195-3adafc86bfd7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:09.24004099Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1373,"client":"0","":"downstairs","session_id":"c618c25c-4af6-4008-b195-3adafc86bfd7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:09.24005091Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1373,"client":"1","":"downstairs","session_id":"c618c25c-4af6-4008-b195-3adafc86bfd7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:09.2400605Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1373,"client":"2","":"downstairs","session_id":"c618c25c-4af6-4008-b195-3adafc86bfd7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:09.24007034Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1373,"":"io task","client":"2","":"downstairs","session_id":"c618c25c-4af6-4008-b195-3adafc86bfd7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:09.240080249Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1373,"":"io task","client":"0","":"downstairs","session_id":"c618c25c-4af6-4008-b195-3adafc86bfd7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:09.240094349Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1373,"":"io task","client":"1","":"downstairs","session_id":"c618c25c-4af6-4008-b195-3adafc86bfd7"} {"msg":"ds_connection connected from Ok(127.0.0.1:65265)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:09.240131738Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1373,"":"io task","client":"2","":"downstairs","session_id":"c618c25c-4af6-4008-b195-3adafc86bfd7"} {"msg":"ds_connection connected from Ok(127.0.0.1:39902)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:09.240142437Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1373,"":"io task","client":"1","":"downstairs","session_id":"c618c25c-4af6-4008-b195-3adafc86bfd7"} {"msg":"ds_connection connected from Ok(127.0.0.1:56239)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:09.240170147Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1373,"":"io task","client":"0","":"downstairs","session_id":"c618c25c-4af6-4008-b195-3adafc86bfd7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:09.240884226Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1373,"client":"1","":"downstairs","session_id":"c618c25c-4af6-4008-b195-3adafc86bfd7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:09.240897436Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1373,"client":"2","":"downstairs","session_id":"c618c25c-4af6-4008-b195-3adafc86bfd7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:09.240908205Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1373,"client":"0","":"downstairs","session_id":"c618c25c-4af6-4008-b195-3adafc86bfd7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:09.241206387Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1373,"session_id":"c618c25c-4af6-4008-b195-3adafc86bfd7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:09.241218006Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1373,"session_id":"c618c25c-4af6-4008-b195-3adafc86bfd7"} {"msg":"[0]R flush_numbers[0..12]: [80, 95, 95, 94, 94, 92, 94, 94, 95, 95, 94, 89]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:09.241234206Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1373,"":"downstairs","":"downstairs","session_id":"c618c25c-4af6-4008-b195-3adafc86bfd7"} {"msg":"[0]R generation[0..12]: [47, 58, 58, 57, 57, 56, 57, 57, 58, 58, 57, 54]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:09.241244516Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1373,"":"downstairs","":"downstairs","session_id":"c618c25c-4af6-4008-b195-3adafc86bfd7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:09.241255385Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1373,"":"downstairs","":"downstairs","session_id":"c618c25c-4af6-4008-b195-3adafc86bfd7"} {"msg":"[1]R flush_numbers[0..12]: [80, 95, 95, 94, 94, 92, 94, 94, 95, 95, 94, 89]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:09.241265675Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1373,"":"downstairs","":"downstairs","session_id":"c618c25c-4af6-4008-b195-3adafc86bfd7"} {"msg":"[1]R generation[0..12]: [47, 58, 58, 57, 57, 56, 57, 57, 58, 58, 57, 54]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:09.241275905Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1373,"":"downstairs","":"downstairs","session_id":"c618c25c-4af6-4008-b195-3adafc86bfd7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:09.241286804Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1373,"":"downstairs","":"downstairs","session_id":"c618c25c-4af6-4008-b195-3adafc86bfd7"} {"msg":"[2]R flush_numbers[0..12]: [80, 95, 95, 94, 94, 92, 94, 94, 95, 95, 94, 89]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:09.241297184Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1373,"":"downstairs","":"downstairs","session_id":"c618c25c-4af6-4008-b195-3adafc86bfd7"} {"msg":"[2]R generation[0..12]: [47, 58, 58, 57, 57, 56, 57, 57, 58, 58, 57, 54]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:09.241307394Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1373,"":"downstairs","":"downstairs","session_id":"c618c25c-4af6-4008-b195-3adafc86bfd7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:09.241318094Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1373,"":"downstairs","":"downstairs","session_id":"c618c25c-4af6-4008-b195-3adafc86bfd7"} {"msg":"Max found gen is 60","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:09.241327313Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1373,"":"downstairs","":"downstairs","session_id":"c618c25c-4af6-4008-b195-3adafc86bfd7"} {"msg":"Generation requested: 60 >= found:60","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:09.241336873Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1373,"":"downstairs","":"downstairs","session_id":"c618c25c-4af6-4008-b195-3adafc86bfd7"} {"msg":"Next flush: 97","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:09.241345963Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1373,"":"downstairs","":"downstairs","session_id":"c618c25c-4af6-4008-b195-3adafc86bfd7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:09.241355382Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1373,"":"downstairs","":"downstairs","session_id":"c618c25c-4af6-4008-b195-3adafc86bfd7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:09.241366472Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1373,"session_id":"c618c25c-4af6-4008-b195-3adafc86bfd7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:09.241375152Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1373,"client":"0","":"downstairs","session_id":"c618c25c-4af6-4008-b195-3adafc86bfd7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:09.241384742Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1373,"client":"1","":"downstairs","session_id":"c618c25c-4af6-4008-b195-3adafc86bfd7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:09.241394301Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1373,"client":"2","":"downstairs","session_id":"c618c25c-4af6-4008-b195-3adafc86bfd7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:09.241403941Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1373,"session_id":"c618c25c-4af6-4008-b195-3adafc86bfd7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:09.241412881Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1373,"session_id":"c618c25c-4af6-4008-b195-3adafc86bfd7"} {"msg":"c243d458-51c1-48ad-b110-62cb4006c091 is now active with session: c618c25c-4af6-4008-b195-3adafc86bfd7","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:09.241422581Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1373,"session_id":"c618c25c-4af6-4008-b195-3adafc86bfd7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:09.24143113Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1373,"session_id":"c618c25c-4af6-4008-b195-3adafc86bfd7"} {"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:09.24144041Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1373} {"msg":"Activated sub_volume c243d458-51c1-48ad-b110-62cb4006c091","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:09.24144827Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1373} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:09.24145625Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1373,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:09.241749041Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1373,"task":"crutest"} 01/30 Write block 132 len 20480 data: 8 9 9 8 10 02/30 Write block 285 len 40960 data: 6 4 4 3 4 4 4 5 6 7 03/30 Write block 420 len 16384 data: 4 4 4 4 04/30 Flush 05/30 Write block 229 len 12288 data: 8 7 7 06/30 Read block 341 len 24576 07/30 Write block 2 len 20480 data: 4 3 5 7 9 08/30 Read block 232 len 24576 09/30 Read block 118 len 24576 10/30 Write block 429 len 12288 data: 3 4 3 11/30 Read block 399 len 36864 12/30 Write block 362 len 32768 data: 7 8 10 10 8 8 7 6 13/30 Read block 76 len 12288 14/30 Write block 535 len 24576 data: 11 9 9 9 9 9 15/30 Flush 16/30 Write block 434 len 12288 data: 3 5 4 17/30 Write block 376 len 20480 data: 7 7 8 9 8 18/30 Read block 521 len 32768 19/30 Read block 11 len 28672 20/30 Write block 436 len 12288 data: 5 4 4 21/30 Read block 468 len 36864 22/30 Read block 63 len 8192 23/30 Read block 542 len 36864 24/30 Write block 418 len 32768 data: 3 4 5 5 5 5 4 3 25/30 Write block 11 len 32768 data: 5 5 5 5 5 6 7 7 26/30 Write block 385 len 32768 data: 5 4 4 4 3 3 3 3 27/30 Write block 319 len 24576 data: 3 2 4 5 6 7 28/30 Write block 546 len 40960 data: 5 5 7 7 8 4 6 7 4 4 29/30 Write block 247 len 12288 data: 6 7 6 30/30 Read block 415 len 40960 ---------------------------------------------------------------- Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Write 3 Done Done Done false 1005 Acked Read 6 Done Done Done false 1006 Acked Write 5 Done Done Done false 1007 Acked Read 6 Sent Done Done false 1008 Acked Read 6 Sent Done Done false 1009 Acked Write 3 Sent Done Done false 1010 Acked Read 9 Sent Done Done false 1011 Acked Write 8 Sent Done Done false 1012 Acked Read 3 Sent Done Done false 1013 Acked Write 6 Sent Done Done false 1014 Acked Flush 0 Sent Done Done false 1015 Acked Write 3 Sent Done Done false 1016 Acked Write 5 Sent Done Done false 1017 Acked Read 8 Sent Done Done false 1018 Acked Read 7 Sent Done Done false 1019 Acked Write 3 Sent Done Done false 1020 Acked Read 9 Sent Done Done false 1021 Acked Read 2 Sent Done Done false 1022 Acked Read 9 Sent Done Done false 1023 Acked Write 8 Sent Done Done false 1024 Acked Write 8 Sent Done Done false 1025 Acked Write 8 Sent Done Done false 1026 Acked Write 6 Sent Done Done false 1027 Acked Write 10 Sent Done Done false 1028 Acked Write 3 Sent Done Done false 1029 Acked Read 10 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 23 0 0 23 Done 3 26 26 55 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1003)) Some(JobId(1014)) Some(JobId(1014)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:44:09.250406941Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1373} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:09.250582446Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1373,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 47 60 60 80 98 98 T T T <--- 1 020-039 58 58 58 95 95 95 F F F 2 040-059 58 58 58 95 95 95 F F F 3 060-079 57 57 57 94 94 94 F F F 4 080-099 57 57 57 94 94 94 F F F 5 100-119 56 56 56 92 92 92 F F F 6 120-139 60 60 60 97 97 97 F F F 7 140-159 57 57 57 94 94 94 F F F 8 160-179 58 58 58 95 95 95 F F F 9 180-199 58 58 58 95 95 95 F F F 10 200-219 57 57 57 94 94 94 F F F 11 220-239 54 60 60 89 98 98 T F F <--- 12 240-259 58 58 58 95 95 95 F T T <--- 13 260-279 59 59 59 96 96 96 F F F 14 280-299 60 60 60 97 97 97 F F F 15 300-319 59 59 59 96 96 96 F T T <--- 16 320-339 57 57 57 94 94 94 F T T <--- 17 340-359 54 54 54 90 90 90 F F F 18 360-379 54 60 60 90 98 98 F T T <--- 19 380-399 59 59 59 96 96 96 F T T <--- 20 400-419 52 52 52 86 86 86 F T T <--- 21 420-439 60 60 60 97 98 98 F T T <--- 22 440-459 58 58 58 95 95 95 F F F 23 460-479 52 52 52 86 86 86 F F F 24 480-499 58 58 58 95 95 95 F F F 25 500-519 58 58 58 95 95 95 F F F 26 520-539 57 60 60 94 98 98 F F F <--- 27 540-559 56 60 60 92 98 98 F T T <--- 28 560-579 59 59 59 96 96 96 F F F 29 580-599 48 48 48 81 81 81 F F F Max gen: 60, Max flush: 98 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 30 Verifying data now No change after verify Loop: 30 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 61 61 61 99 99 99 F F F 1 020-039 58 58 58 95 95 95 F F F 2 040-059 58 58 58 95 95 95 F F F 3 060-079 57 57 57 94 94 94 F F F 4 080-099 57 57 57 94 94 94 F F F 5 100-119 56 56 56 92 92 92 F F F 6 120-139 60 60 60 97 97 97 F F F 7 140-159 57 57 57 94 94 94 F F F 8 160-179 58 58 58 95 95 95 F F F 9 180-199 58 58 58 95 95 95 F F F 10 200-219 57 57 57 94 94 94 F F F 11 220-239 60 60 60 98 98 98 F F F 12 240-259 61 61 61 99 99 99 F F F 13 260-279 59 59 59 96 96 96 F F F 14 280-299 60 60 60 97 97 97 F F F 15 300-319 61 61 61 99 99 99 F F F 16 320-339 61 61 61 99 99 99 F F F 17 340-359 54 54 54 90 90 90 F F F 18 360-379 61 61 61 99 99 99 F F F 19 380-399 61 61 61 99 99 99 F F F 20 400-419 61 61 61 99 99 99 F F F 21 420-439 61 61 61 99 99 99 F F F 22 440-459 58 58 58 95 95 95 F F F 23 460-479 52 52 52 86 86 86 F F F 24 480-499 58 58 58 95 95 95 F F F 25 500-519 58 58 58 95 95 95 F F F 26 520-539 60 60 60 98 98 98 F F F 27 540-559 61 61 61 99 99 99 F F F 28 560-579 59 59 59 96 96 96 F F F 29 580-599 48 48 48 81 81 81 F F F Max gen: 61, Max flush: 99 Begin loop 31 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:13.502964388Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1381,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:13.511120943Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1381,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:13.511407804Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1381} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:13.511433444Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1381} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:13.511441863Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1381} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:13.511455293Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1381} {"msg":"Crucible 890c6191-85a9-4d0b-b143-1474b2bdd884 has session id: e54a6a92-e09b-4811-b4d4-bae4c240c494","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:13.511855832Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1381,"session_id":"e54a6a92-e09b-4811-b4d4-bae4c240c494"} {"msg":"Upstairs opts: Upstairs UUID: 890c6191-85a9-4d0b-b143-1474b2bdd884, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:13.511886821Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1381,"session_id":"e54a6a92-e09b-4811-b4d4-bae4c240c494"} {"msg":"Crucible stats registered with UUID: 890c6191-85a9-4d0b-b143-1474b2bdd884","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:13.51189711Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1381,"session_id":"e54a6a92-e09b-4811-b4d4-bae4c240c494"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:44:13.51190628Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1381,"session_id":"e54a6a92-e09b-4811-b4d4-bae4c240c494"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:13.512045016Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1381,"":"io task","client":"2","":"downstairs","session_id":"e54a6a92-e09b-4811-b4d4-bae4c240c494"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:13.512076005Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1381,"":"io task","client":"1","":"downstairs","session_id":"e54a6a92-e09b-4811-b4d4-bae4c240c494"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:13.512087025Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1381,"":"io task","client":"0","":"downstairs","session_id":"e54a6a92-e09b-4811-b4d4-bae4c240c494"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:13.512219511Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1381,"task":"crutest"} {"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:13.512235281Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1381} {"msg":"890c6191-85a9-4d0b-b143-1474b2bdd884 active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:13.51226953Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1381,"session_id":"e54a6a92-e09b-4811-b4d4-bae4c240c494"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:13.512279009Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1381,"client":"0","":"downstairs","session_id":"e54a6a92-e09b-4811-b4d4-bae4c240c494"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:13.512288769Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1381,"client":"1","":"downstairs","session_id":"e54a6a92-e09b-4811-b4d4-bae4c240c494"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:13.512298409Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1381,"client":"2","":"downstairs","session_id":"e54a6a92-e09b-4811-b4d4-bae4c240c494"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:13.512307848Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1381,"":"io task","client":"2","":"downstairs","session_id":"e54a6a92-e09b-4811-b4d4-bae4c240c494"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:13.512319588Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1381,"":"io task","client":"0","":"downstairs","session_id":"e54a6a92-e09b-4811-b4d4-bae4c240c494"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:13.512329608Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1381,"":"io task","client":"1","":"downstairs","session_id":"e54a6a92-e09b-4811-b4d4-bae4c240c494"} {"msg":"ds_connection connected from Ok(127.0.0.1:42785)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:13.512355087Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1381,"":"io task","client":"2","":"downstairs","session_id":"e54a6a92-e09b-4811-b4d4-bae4c240c494"} {"msg":"ds_connection connected from Ok(127.0.0.1:63956)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:13.512397456Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1381,"":"io task","client":"0","":"downstairs","session_id":"e54a6a92-e09b-4811-b4d4-bae4c240c494"} {"msg":"ds_connection connected from Ok(127.0.0.1:50007)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:13.512408376Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1381,"":"io task","client":"1","":"downstairs","session_id":"e54a6a92-e09b-4811-b4d4-bae4c240c494"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:13.512993549Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1381,"client":"2","":"downstairs","session_id":"e54a6a92-e09b-4811-b4d4-bae4c240c494"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:13.513051827Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1381,"client":"0","":"downstairs","session_id":"e54a6a92-e09b-4811-b4d4-bae4c240c494"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:13.513129375Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1381,"client":"1","":"downstairs","session_id":"e54a6a92-e09b-4811-b4d4-bae4c240c494"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:13.51329097Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1381,"session_id":"e54a6a92-e09b-4811-b4d4-bae4c240c494"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:13.51330892Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1381,"session_id":"e54a6a92-e09b-4811-b4d4-bae4c240c494"} {"msg":"[0]R flush_numbers[0..12]: [99, 95, 95, 94, 94, 92, 97, 94, 95, 95, 94, 98]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:13.513460435Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1381,"":"downstairs","":"downstairs","session_id":"e54a6a92-e09b-4811-b4d4-bae4c240c494"} {"msg":"[0]R generation[0..12]: [61, 58, 58, 57, 57, 56, 60, 57, 58, 58, 57, 60]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:13.513471535Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1381,"":"downstairs","":"downstairs","session_id":"e54a6a92-e09b-4811-b4d4-bae4c240c494"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:13.513482605Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1381,"":"downstairs","":"downstairs","session_id":"e54a6a92-e09b-4811-b4d4-bae4c240c494"} {"msg":"[1]R flush_numbers[0..12]: [99, 95, 95, 94, 94, 92, 97, 94, 95, 95, 94, 98]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:13.513492994Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1381,"":"downstairs","":"downstairs","session_id":"e54a6a92-e09b-4811-b4d4-bae4c240c494"} {"msg":"[1]R generation[0..12]: [61, 58, 58, 57, 57, 56, 60, 57, 58, 58, 57, 60]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:13.513503264Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1381,"":"downstairs","":"downstairs","session_id":"e54a6a92-e09b-4811-b4d4-bae4c240c494"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:13.513516094Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1381,"":"downstairs","":"downstairs","session_id":"e54a6a92-e09b-4811-b4d4-bae4c240c494"} {"msg":"[2]R flush_numbers[0..12]: [99, 95, 95, 94, 94, 92, 97, 94, 95, 95, 94, 98]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:13.513526283Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1381,"":"downstairs","":"downstairs","session_id":"e54a6a92-e09b-4811-b4d4-bae4c240c494"} {"msg":"[2]R generation[0..12]: [61, 58, 58, 57, 57, 56, 60, 57, 58, 58, 57, 60]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:13.513536373Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1381,"":"downstairs","":"downstairs","session_id":"e54a6a92-e09b-4811-b4d4-bae4c240c494"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:13.513547243Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1381,"":"downstairs","":"downstairs","session_id":"e54a6a92-e09b-4811-b4d4-bae4c240c494"} {"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:13.513556662Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1381,"":"downstairs","":"downstairs","session_id":"e54a6a92-e09b-4811-b4d4-bae4c240c494"} {"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:13.513566272Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1381,"":"downstairs","":"downstairs","session_id":"e54a6a92-e09b-4811-b4d4-bae4c240c494"} {"msg":"Next flush: 100","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:13.513575492Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1381,"":"downstairs","":"downstairs","session_id":"e54a6a92-e09b-4811-b4d4-bae4c240c494"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:13.513584832Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1381,"":"downstairs","":"downstairs","session_id":"e54a6a92-e09b-4811-b4d4-bae4c240c494"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:13.513594371Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1381,"session_id":"e54a6a92-e09b-4811-b4d4-bae4c240c494"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:13.513603071Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1381,"client":"0","":"downstairs","session_id":"e54a6a92-e09b-4811-b4d4-bae4c240c494"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:13.513612711Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1381,"client":"1","":"downstairs","session_id":"e54a6a92-e09b-4811-b4d4-bae4c240c494"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:13.513622241Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1381,"client":"2","":"downstairs","session_id":"e54a6a92-e09b-4811-b4d4-bae4c240c494"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:13.51363208Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1381,"session_id":"e54a6a92-e09b-4811-b4d4-bae4c240c494"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:13.51364119Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1381,"session_id":"e54a6a92-e09b-4811-b4d4-bae4c240c494"} {"msg":"890c6191-85a9-4d0b-b143-1474b2bdd884 is now active with session: e54a6a92-e09b-4811-b4d4-bae4c240c494","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:13.5136511Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1381,"session_id":"e54a6a92-e09b-4811-b4d4-bae4c240c494"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:13.513661519Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1381,"session_id":"e54a6a92-e09b-4811-b4d4-bae4c240c494"} {"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:13.513670489Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1381} {"msg":"Activated sub_volume 890c6191-85a9-4d0b-b143-1474b2bdd884","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:13.513678319Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1381} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:13.513686229Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1381,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:13.51398864Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1381,"task":"crutest"} 01/30 Write block 541 len 20480 data: 6 5 5 6 6 02/30 Flush 03/30 Write block 187 len 24576 data: 4 3 4 3 3 3 04/30 Write block 352 len 12288 data: 3 3 4 05/30 Write block 212 len 16384 data: 8 8 8 6 06/30 Read block 130 len 24576 07/30 Flush 08/30 Write block 114 len 4096 data: 4 09/30 Write block 357 len 28672 data: 5 5 4 6 6 8 9 10/30 Read block 315 len 40960 11/30 Read block 13 len 20480 12/30 Read block 590 len 36864 13/30 Write block 186 len 36864 data: 4 5 4 5 4 4 4 3 4 14/30 Flush 15/30 Flush 16/30 Write block 54 len 24576 data: 6 7 6 6 6 4 17/30 Write block 581 len 32768 data: 3 4 4 4 4 4 5 6 18/30 Write block 173 len 12288 data: 7 6 7 19/30 Read block 512 len 36864 20/30 Read block 124 len 28672 21/30 Read block 203 len 28672 22/30 Write block 247 len 36864 data: 7 8 7 7 4 4 6 6 7 23/30 Write block 384 len 40960 data: 6 6 5 5 5 4 4 4 4 3 24/30 Read block 179 len 32768 25/30 Write block 488 len 28672 data: 5 4 5 6 4 4 3 26/30 Write block 468 len 12288 data: 3 4 4 27/30 Read block 41 len 28672 28/30 Read block 234 len 4096 29/30 Write block 159 len 8192 data: 5 6 30/30 Write block 232 len 8192 data: 6 6 ---------------------------------------------------------------- Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Sent Done false 1001 Acked Flush 0 Done Sent Done false 1002 Acked Write 6 Done Sent Done false 1003 Acked Write 3 Done Sent Done false 1004 Acked Write 4 Done Sent Done false 1005 Acked Read 6 Done Sent Done false 1006 Acked Flush 0 Done Sent Done false 1007 Acked Write 1 Done Sent Done false 1008 Acked Write 7 Done Sent Done false 1009 Acked Read 10 Done Sent Done false 1010 Acked Read 5 Done Sent Done false 1011 Acked Read 9 Done Sent Done false 1012 Acked Write 9 Done Sent Done false 1013 Acked Flush 0 Done Sent Done false 1014 Acked Flush 0 Done Sent Done false 1015 Acked Write 6 Done Sent Done false 1016 Acked Write 8 Done Sent Done false 1017 Acked Write 3 Done Sent Done false 1018 Acked Read 9 Done Sent Done false 1019 Acked Read 7 Done Sent Done false 1020 Acked Read 7 Done Sent Done false 1021 Acked Write 9 Done Sent Done false 1022 Acked Write 10 Done Sent Done false 1023 Acked Read 8 Done Sent Done false 1024 Acked Write 7 Done Sent Done false 1025 Acked Write 3 Done Sent Done false 1026 Acked Read 7 Done Sent Done false 1027 Acked Read 1 Done Sent Done false 1028 Acked Write 2 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 30 2 34 Done 28 0 28 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) None Some(JobId(1014)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:44:13.522613401Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1381} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:13.522810186Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1381,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 61 61 61 99 99 99 F F F 1 020-039 58 58 58 95 95 95 F F F 2 040-059 58 58 58 95 95 95 T F T <--- 3 060-079 57 57 57 94 94 94 F F F 4 080-099 57 57 57 94 94 94 F F F 5 100-119 62 56 62 102 92 102 F F F <--- 6 120-139 60 60 60 97 97 97 F F F 7 140-159 57 57 57 94 94 94 T F T <--- 8 160-179 58 58 58 95 95 95 T F T <--- 9 180-199 62 58 62 102 95 102 F F F <--- 10 200-219 62 57 62 101 94 101 F F F <--- 11 220-239 60 60 60 98 98 98 T F T <--- 12 240-259 61 61 61 99 99 99 T F T <--- 13 260-279 59 59 59 96 96 96 F F F 14 280-299 60 60 60 97 97 97 F F F 15 300-319 61 61 61 99 99 99 F F F 16 320-339 61 61 61 99 99 99 F F F 17 340-359 62 54 62 102 90 102 F F F <--- 18 360-379 62 61 62 102 99 102 F F F <--- 19 380-399 61 61 61 99 99 99 T F T <--- 20 400-419 61 61 61 99 99 99 F F F 21 420-439 61 61 61 99 99 99 F F F 22 440-459 58 58 58 95 95 95 F F F 23 460-479 52 52 52 86 86 86 T F T <--- 24 480-499 58 58 58 95 95 95 T F T <--- 25 500-519 58 58 58 95 95 95 F F F 26 520-539 60 60 60 98 98 98 F F F 27 540-559 62 61 62 100 99 100 F F F <--- 28 560-579 59 59 59 96 96 96 F F F 29 580-599 48 48 48 81 81 81 T F T <--- Max gen: 62, Max flush: 102 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 31 Verifying data now No change after verify Loop: 31 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 61 61 61 99 99 99 F F F 1 020-039 58 58 58 95 95 95 F F F 2 040-059 63 63 63 103 103 103 F F F 3 060-079 57 57 57 94 94 94 F F F 4 080-099 57 57 57 94 94 94 F F F 5 100-119 62 62 62 102 102 102 F F F 6 120-139 60 60 60 97 97 97 F F F 7 140-159 63 63 63 103 103 103 F F F 8 160-179 63 63 63 103 103 103 F F F 9 180-199 62 62 62 102 102 102 F F F 10 200-219 62 62 62 101 101 101 F F F 11 220-239 63 63 63 103 103 103 F F F 12 240-259 63 63 63 103 103 103 F F F 13 260-279 59 59 59 96 96 96 F F F 14 280-299 60 60 60 97 97 97 F F F 15 300-319 61 61 61 99 99 99 F F F 16 320-339 61 61 61 99 99 99 F F F 17 340-359 62 62 62 102 102 102 F F F 18 360-379 62 62 62 102 102 102 F F F 19 380-399 63 63 63 103 103 103 F F F 20 400-419 61 61 61 99 99 99 F F F 21 420-439 61 61 61 99 99 99 F F F 22 440-459 58 58 58 95 95 95 F F F 23 460-479 63 63 63 103 103 103 F F F 24 480-499 63 63 63 103 103 103 F F F 25 500-519 58 58 58 95 95 95 F F F 26 520-539 60 60 60 98 98 98 F F F 27 540-559 62 62 62 100 100 100 F F F 28 560-579 59 59 59 96 96 96 F F F 29 580-599 63 63 63 103 103 103 F F F Max gen: 63, Max flush: 103 Begin loop 32 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:17.775271325Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1389,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:17.783266874Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1389,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:17.783552156Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1389} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:17.783577065Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1389} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:17.783585895Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1389} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:17.783597445Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1389} {"msg":"Crucible d4d69a66-4efc-4774-b873-dcdcf24ba3f8 has session id: 91a07224-ffa4-4514-8666-87589db0430a","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:17.783958424Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1389,"session_id":"91a07224-ffa4-4514-8666-87589db0430a"} {"msg":"Upstairs opts: Upstairs UUID: d4d69a66-4efc-4774-b873-dcdcf24ba3f8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:17.783986973Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1389,"session_id":"91a07224-ffa4-4514-8666-87589db0430a"} {"msg":"Crucible stats registered with UUID: d4d69a66-4efc-4774-b873-dcdcf24ba3f8","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:17.784006893Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1389,"session_id":"91a07224-ffa4-4514-8666-87589db0430a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:44:17.784016373Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1389,"session_id":"91a07224-ffa4-4514-8666-87589db0430a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:17.784147389Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1389,"":"io task","client":"2","":"downstairs","session_id":"91a07224-ffa4-4514-8666-87589db0430a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:17.784178598Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1389,"":"io task","client":"0","":"downstairs","session_id":"91a07224-ffa4-4514-8666-87589db0430a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:17.784189318Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1389,"":"io task","client":"1","":"downstairs","session_id":"91a07224-ffa4-4514-8666-87589db0430a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:17.784327344Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1389,"task":"crutest"} {"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:17.784341683Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1389} {"msg":"d4d69a66-4efc-4774-b873-dcdcf24ba3f8 active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:17.784376542Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1389,"session_id":"91a07224-ffa4-4514-8666-87589db0430a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:17.784385952Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1389,"client":"0","":"downstairs","session_id":"91a07224-ffa4-4514-8666-87589db0430a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:17.784395752Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1389,"client":"1","":"downstairs","session_id":"91a07224-ffa4-4514-8666-87589db0430a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:17.784405421Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1389,"client":"2","":"downstairs","session_id":"91a07224-ffa4-4514-8666-87589db0430a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:17.784414971Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1389,"":"io task","client":"2","":"downstairs","session_id":"91a07224-ffa4-4514-8666-87589db0430a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:17.784424951Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1389,"":"io task","client":"0","":"downstairs","session_id":"91a07224-ffa4-4514-8666-87589db0430a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:17.784434881Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1389,"":"io task","client":"1","":"downstairs","session_id":"91a07224-ffa4-4514-8666-87589db0430a"} {"msg":"ds_connection connected from Ok(127.0.0.1:35990)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:17.784505839Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1389,"":"io task","client":"0","":"downstairs","session_id":"91a07224-ffa4-4514-8666-87589db0430a"} {"msg":"ds_connection connected from Ok(127.0.0.1:47921)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:17.784632595Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1389,"":"io task","client":"1","":"downstairs","session_id":"91a07224-ffa4-4514-8666-87589db0430a"} {"msg":"ds_connection connected from Ok(127.0.0.1:34489)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:17.784658114Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1389,"":"io task","client":"2","":"downstairs","session_id":"91a07224-ffa4-4514-8666-87589db0430a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:17.785167179Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1389,"client":"2","":"downstairs","session_id":"91a07224-ffa4-4514-8666-87589db0430a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:17.785196349Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1389,"client":"0","":"downstairs","session_id":"91a07224-ffa4-4514-8666-87589db0430a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:17.785269716Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1389,"client":"1","":"downstairs","session_id":"91a07224-ffa4-4514-8666-87589db0430a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:17.785524049Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1389,"session_id":"91a07224-ffa4-4514-8666-87589db0430a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:17.785533709Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1389,"session_id":"91a07224-ffa4-4514-8666-87589db0430a"} {"msg":"[0]R flush_numbers[0..12]: [99, 95, 103, 94, 94, 102, 97, 103, 103, 102, 101, 103]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:17.785604327Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1389,"":"downstairs","":"downstairs","session_id":"91a07224-ffa4-4514-8666-87589db0430a"} {"msg":"[0]R generation[0..12]: [61, 58, 63, 57, 57, 62, 60, 63, 63, 62, 62, 63]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:17.785615337Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1389,"":"downstairs","":"downstairs","session_id":"91a07224-ffa4-4514-8666-87589db0430a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:17.785626366Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1389,"":"downstairs","":"downstairs","session_id":"91a07224-ffa4-4514-8666-87589db0430a"} {"msg":"[1]R flush_numbers[0..12]: [99, 95, 103, 94, 94, 102, 97, 103, 103, 102, 101, 103]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:17.785636826Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1389,"":"downstairs","":"downstairs","session_id":"91a07224-ffa4-4514-8666-87589db0430a"} {"msg":"[1]R generation[0..12]: [61, 58, 63, 57, 57, 62, 60, 63, 63, 62, 62, 63]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:17.785647216Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1389,"":"downstairs","":"downstairs","session_id":"91a07224-ffa4-4514-8666-87589db0430a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:17.785658065Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1389,"":"downstairs","":"downstairs","session_id":"91a07224-ffa4-4514-8666-87589db0430a"} {"msg":"[2]R flush_numbers[0..12]: [99, 95, 103, 94, 94, 102, 97, 103, 103, 102, 101, 103]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:17.785668515Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1389,"":"downstairs","":"downstairs","session_id":"91a07224-ffa4-4514-8666-87589db0430a"} {"msg":"[2]R generation[0..12]: [61, 58, 63, 57, 57, 62, 60, 63, 63, 62, 62, 63]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:17.785678735Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1389,"":"downstairs","":"downstairs","session_id":"91a07224-ffa4-4514-8666-87589db0430a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:17.785689544Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1389,"":"downstairs","":"downstairs","session_id":"91a07224-ffa4-4514-8666-87589db0430a"} {"msg":"Max found gen is 64","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:17.785698904Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1389,"":"downstairs","":"downstairs","session_id":"91a07224-ffa4-4514-8666-87589db0430a"} {"msg":"Generation requested: 64 >= found:64","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:17.785709934Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1389,"":"downstairs","":"downstairs","session_id":"91a07224-ffa4-4514-8666-87589db0430a"} {"msg":"Next flush: 104","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:17.785719184Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1389,"":"downstairs","":"downstairs","session_id":"91a07224-ffa4-4514-8666-87589db0430a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:17.785728533Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1389,"":"downstairs","":"downstairs","session_id":"91a07224-ffa4-4514-8666-87589db0430a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:17.785738513Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1389,"session_id":"91a07224-ffa4-4514-8666-87589db0430a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:17.785747283Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1389,"client":"0","":"downstairs","session_id":"91a07224-ffa4-4514-8666-87589db0430a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:17.785757162Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1389,"client":"1","":"downstairs","session_id":"91a07224-ffa4-4514-8666-87589db0430a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:17.785766912Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1389,"client":"2","":"downstairs","session_id":"91a07224-ffa4-4514-8666-87589db0430a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:17.785776832Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1389,"session_id":"91a07224-ffa4-4514-8666-87589db0430a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:17.785786042Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1389,"session_id":"91a07224-ffa4-4514-8666-87589db0430a"} {"msg":"d4d69a66-4efc-4774-b873-dcdcf24ba3f8 is now active with session: 91a07224-ffa4-4514-8666-87589db0430a","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:17.785795891Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1389,"session_id":"91a07224-ffa4-4514-8666-87589db0430a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:17.785804471Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1389,"session_id":"91a07224-ffa4-4514-8666-87589db0430a"} {"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:17.785813551Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1389} {"msg":"Activated sub_volume d4d69a66-4efc-4774-b873-dcdcf24ba3f8","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:17.785821491Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1389} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:17.7858294Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1389,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:17.786134082Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1389,"task":"crutest"} 01/30 Write block 474 len 24576 data: 2 5 7 7 7 7 02/30 Write block 182 len 28672 data: 3 3 5 4 5 6 5 03/30 Write block 217 len 16384 data: 8 6 5 4 04/30 Write block 568 len 16384 data: 5 4 6 6 05/30 Read block 498 len 20480 06/30 Read block 345 len 12288 07/30 Flush 08/30 Write block 265 len 28672 data: 2 2 3 4 4 4 6 09/30 Read block 97 len 16384 10/30 Read block 256 len 28672 11/30 Write block 495 len 8192 data: 5 6 12/30 Write block 112 len 40960 data: 5 6 5 4 6 5 4 6 6 6 13/30 Write block 225 len 12288 data: 10 10 9 14/30 Write block 435 len 8192 data: 6 6 15/30 Write block 582 len 16384 data: 5 5 5 5 16/30 Read block 160 len 40960 17/30 Read block 76 len 36864 18/30 Read block 146 len 32768 19/30 Write block 56 len 28672 data: 7 7 7 5 4 6 6 20/30 Read block 174 len 28672 21/30 Read block 279 len 24576 22/30 Flush 23/30 Write block 50 len 8192 data: 3 4 24/30 Flush 25/30 Write block 560 len 40960 data: 8 7 8 6 7 7 7 4 6 5 26/30 Read block 514 len 32768 27/30 Write block 475 len 16384 data: 6 8 8 8 28/30 Write block 324 len 40960 data: 8 7 6 6 5 5 6 6 5 5 29/30 Read block 5 len 24576 30/30 Read block 591 len 20480 ---------------------------------------------------------------- Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Sent Done false 1001 Acked Write 7 Done Sent Done false 1002 Acked Write 4 Done Sent Done false 1003 Acked Write 4 Done Sent Done false 1004 Acked Read 5 Done Sent Done false 1005 Acked Read 3 Done Sent Done false 1006 Acked Flush 0 Done Sent Done false 1007 Acked Write 7 Done Sent Done false 1008 Acked Read 4 Done Sent Done false 1009 Acked Read 7 Done Sent Done false 1010 Acked Write 2 Done Sent Done false 1011 Acked Write 10 Done Sent Done false 1012 Acked Write 3 Done Sent Done false 1013 Acked Write 2 Done Sent Done false 1014 Acked Write 4 Done Sent Done false 1015 Acked Read 10 Done Sent Done false 1016 Acked Read 9 Done Sent Done false 1017 Acked Read 8 Done Sent Done false 1018 Acked Write 7 Done Sent Done false 1019 Acked Read 7 Done Sent Done false 1020 Acked Read 6 Done Sent Done false 1021 Acked Flush 0 Done Sent Done false 1022 Acked Write 2 Done Sent Done false 1023 Acked Flush 0 Done Sent Done false 1024 Acked Write 10 Done Sent Done false 1025 Acked Read 8 Done Sent Done false 1026 Acked Write 4 Done Sent Done false 1027 Acked Write 10 Done Sent Done false 1028 Acked Read 6 Done Sent Done false 1029 Acked Read 5 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 30 0 30 Done 30 0 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) None Some(JobId(1023)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:44:17.794948727Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1389} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:17.795110873Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1389,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 61 61 61 99 99 99 F F F 1 020-039 58 58 58 95 95 95 F F F 2 040-059 64 63 64 106 103 106 F F F <--- 3 060-079 64 57 64 105 94 105 F F F <--- 4 080-099 57 57 57 94 94 94 F F F 5 100-119 64 62 64 105 102 105 F F F <--- 6 120-139 64 60 64 105 97 105 F F F <--- 7 140-159 63 63 63 103 103 103 F F F 8 160-179 63 63 63 103 103 103 F F F 9 180-199 64 62 64 104 102 104 F F F <--- 10 200-219 64 62 64 104 101 104 F F F <--- 11 220-239 64 63 64 105 103 105 F F F <--- 12 240-259 63 63 63 103 103 103 F F F 13 260-279 64 59 64 105 96 105 F F F <--- 14 280-299 60 60 60 97 97 97 F F F 15 300-319 61 61 61 99 99 99 F F F 16 320-339 61 61 61 99 99 99 T F T <--- 17 340-359 62 62 62 102 102 102 F F F 18 360-379 62 62 62 102 102 102 F F F 19 380-399 63 63 63 103 103 103 F F F 20 400-419 61 61 61 99 99 99 F F F 21 420-439 64 61 64 105 99 105 F F F <--- 22 440-459 58 58 58 95 95 95 F F F 23 460-479 64 63 64 104 103 104 T F T <--- 24 480-499 64 63 64 105 103 105 F F F <--- 25 500-519 58 58 58 95 95 95 F F F 26 520-539 60 60 60 98 98 98 F F F 27 540-559 62 62 62 100 100 100 F F F 28 560-579 64 59 64 104 96 104 T F T <--- 29 580-599 64 63 64 105 103 105 F F F <--- Max gen: 64, Max flush: 106 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 32 Verifying data now No change after verify Loop: 32 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 61 61 61 99 99 99 F F F 1 020-039 58 58 58 95 95 95 F F F 2 040-059 64 64 64 106 106 106 F F F 3 060-079 64 64 64 105 105 105 F F F 4 080-099 57 57 57 94 94 94 F F F 5 100-119 64 64 64 105 105 105 F F F 6 120-139 64 64 64 105 105 105 F F F 7 140-159 63 63 63 103 103 103 F F F 8 160-179 63 63 63 103 103 103 F F F 9 180-199 64 64 64 104 104 104 F F F 10 200-219 64 64 64 104 104 104 F F F 11 220-239 64 64 64 105 105 105 F F F 12 240-259 63 63 63 103 103 103 F F F 13 260-279 64 64 64 105 105 105 F F F 14 280-299 60 60 60 97 97 97 F F F 15 300-319 61 61 61 99 99 99 F F F 16 320-339 65 65 65 107 107 107 F F F 17 340-359 62 62 62 102 102 102 F F F 18 360-379 62 62 62 102 102 102 F F F 19 380-399 63 63 63 103 103 103 F F F 20 400-419 61 61 61 99 99 99 F F F 21 420-439 64 64 64 105 105 105 F F F 22 440-459 58 58 58 95 95 95 F F F 23 460-479 65 65 65 107 107 107 F F F 24 480-499 64 64 64 105 105 105 F F F 25 500-519 58 58 58 95 95 95 F F F 26 520-539 60 60 60 98 98 98 F F F 27 540-559 62 62 62 100 100 100 F F F 28 560-579 65 65 65 107 107 107 F F F 29 580-599 64 64 64 105 105 105 F F F Max gen: 65, Max flush: 107 Begin loop 33 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:22.046102014Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1397,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:44:22.053100362Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1397,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:22.05315513Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1397,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:22.060009903Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1397,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:22.060279425Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1397} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:22.060305364Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1397} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:22.060313694Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1397} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:22.060325124Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1397} {"msg":"Crucible ea350bc0-2663-4c87-af1e-694f1ab59c4d has session id: 0aeca8b1-faa7-4a6d-a12f-7a31aebe9ffc","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:22.060659084Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1397,"session_id":"0aeca8b1-faa7-4a6d-a12f-7a31aebe9ffc"} {"msg":"Upstairs opts: Upstairs UUID: ea350bc0-2663-4c87-af1e-694f1ab59c4d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:22.060688563Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1397,"session_id":"0aeca8b1-faa7-4a6d-a12f-7a31aebe9ffc"} {"msg":"Crucible stats registered with UUID: ea350bc0-2663-4c87-af1e-694f1ab59c4d","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:22.060699663Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1397,"session_id":"0aeca8b1-faa7-4a6d-a12f-7a31aebe9ffc"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:44:22.060708863Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1397,"session_id":"0aeca8b1-faa7-4a6d-a12f-7a31aebe9ffc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:22.060830659Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1397,"":"io task","client":"1","":"downstairs","session_id":"0aeca8b1-faa7-4a6d-a12f-7a31aebe9ffc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:22.060848878Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1397,"":"io task","client":"0","":"downstairs","session_id":"0aeca8b1-faa7-4a6d-a12f-7a31aebe9ffc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:22.060858918Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1397,"":"io task","client":"2","":"downstairs","session_id":"0aeca8b1-faa7-4a6d-a12f-7a31aebe9ffc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:22.060986165Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1397,"task":"crutest"} {"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:22.061010904Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1397} {"msg":"ea350bc0-2663-4c87-af1e-694f1ab59c4d active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:22.061023713Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1397,"session_id":"0aeca8b1-faa7-4a6d-a12f-7a31aebe9ffc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:22.061032533Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1397,"client":"0","":"downstairs","session_id":"0aeca8b1-faa7-4a6d-a12f-7a31aebe9ffc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:22.061042293Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1397,"client":"1","":"downstairs","session_id":"0aeca8b1-faa7-4a6d-a12f-7a31aebe9ffc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:22.061051843Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1397,"client":"2","":"downstairs","session_id":"0aeca8b1-faa7-4a6d-a12f-7a31aebe9ffc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:22.061061322Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1397,"":"io task","client":"2","":"downstairs","session_id":"0aeca8b1-faa7-4a6d-a12f-7a31aebe9ffc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:22.061071122Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1397,"":"io task","client":"0","":"downstairs","session_id":"0aeca8b1-faa7-4a6d-a12f-7a31aebe9ffc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:22.061080982Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1397,"":"io task","client":"1","":"downstairs","session_id":"0aeca8b1-faa7-4a6d-a12f-7a31aebe9ffc"} {"msg":"ds_connection connected from Ok(127.0.0.1:38622)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:22.0611372Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1397,"":"io task","client":"2","":"downstairs","session_id":"0aeca8b1-faa7-4a6d-a12f-7a31aebe9ffc"} {"msg":"ds_connection connected from Ok(127.0.0.1:51353)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:22.06114841Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1397,"":"io task","client":"1","":"downstairs","session_id":"0aeca8b1-faa7-4a6d-a12f-7a31aebe9ffc"} {"msg":"ds_connection connected from Ok(127.0.0.1:43888)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:22.061252667Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1397,"":"io task","client":"0","":"downstairs","session_id":"0aeca8b1-faa7-4a6d-a12f-7a31aebe9ffc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:22.06184589Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1397,"client":"2","":"downstairs","session_id":"0aeca8b1-faa7-4a6d-a12f-7a31aebe9ffc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:22.061864949Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1397,"client":"1","":"downstairs","session_id":"0aeca8b1-faa7-4a6d-a12f-7a31aebe9ffc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:22.061876279Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1397,"client":"0","":"downstairs","session_id":"0aeca8b1-faa7-4a6d-a12f-7a31aebe9ffc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:22.062160561Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1397,"session_id":"0aeca8b1-faa7-4a6d-a12f-7a31aebe9ffc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:22.06219402Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1397,"session_id":"0aeca8b1-faa7-4a6d-a12f-7a31aebe9ffc"} {"msg":"[0]R flush_numbers[0..12]: [99, 95, 106, 105, 94, 105, 105, 103, 103, 104, 104, 105]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:22.062275077Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1397,"":"downstairs","":"downstairs","session_id":"0aeca8b1-faa7-4a6d-a12f-7a31aebe9ffc"} {"msg":"[0]R generation[0..12]: [61, 58, 64, 64, 57, 64, 64, 63, 63, 64, 64, 64]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:22.062286187Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1397,"":"downstairs","":"downstairs","session_id":"0aeca8b1-faa7-4a6d-a12f-7a31aebe9ffc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:22.062297127Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1397,"":"downstairs","":"downstairs","session_id":"0aeca8b1-faa7-4a6d-a12f-7a31aebe9ffc"} {"msg":"[1]R flush_numbers[0..12]: [99, 95, 106, 105, 94, 105, 105, 103, 103, 104, 104, 105]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:22.062307546Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1397,"":"downstairs","":"downstairs","session_id":"0aeca8b1-faa7-4a6d-a12f-7a31aebe9ffc"} {"msg":"[1]R generation[0..12]: [61, 58, 64, 64, 57, 64, 64, 63, 63, 64, 64, 64]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:22.062317806Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1397,"":"downstairs","":"downstairs","session_id":"0aeca8b1-faa7-4a6d-a12f-7a31aebe9ffc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:22.062328626Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1397,"":"downstairs","":"downstairs","session_id":"0aeca8b1-faa7-4a6d-a12f-7a31aebe9ffc"} {"msg":"[2]R flush_numbers[0..12]: [99, 95, 106, 105, 94, 105, 105, 103, 103, 104, 104, 105]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:22.062339085Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1397,"":"downstairs","":"downstairs","session_id":"0aeca8b1-faa7-4a6d-a12f-7a31aebe9ffc"} {"msg":"[2]R generation[0..12]: [61, 58, 64, 64, 57, 64, 64, 63, 63, 64, 64, 64]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:22.062349245Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1397,"":"downstairs","":"downstairs","session_id":"0aeca8b1-faa7-4a6d-a12f-7a31aebe9ffc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:22.062360115Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1397,"":"downstairs","":"downstairs","session_id":"0aeca8b1-faa7-4a6d-a12f-7a31aebe9ffc"} {"msg":"Max found gen is 66","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:22.062369385Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1397,"":"downstairs","":"downstairs","session_id":"0aeca8b1-faa7-4a6d-a12f-7a31aebe9ffc"} {"msg":"Generation requested: 66 >= found:66","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:22.062378984Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1397,"":"downstairs","":"downstairs","session_id":"0aeca8b1-faa7-4a6d-a12f-7a31aebe9ffc"} {"msg":"Next flush: 108","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:22.062388184Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1397,"":"downstairs","":"downstairs","session_id":"0aeca8b1-faa7-4a6d-a12f-7a31aebe9ffc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:22.062397514Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1397,"":"downstairs","":"downstairs","session_id":"0aeca8b1-faa7-4a6d-a12f-7a31aebe9ffc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:22.062407124Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1397,"session_id":"0aeca8b1-faa7-4a6d-a12f-7a31aebe9ffc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:22.062415733Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1397,"client":"0","":"downstairs","session_id":"0aeca8b1-faa7-4a6d-a12f-7a31aebe9ffc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:22.062426793Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1397,"client":"1","":"downstairs","session_id":"0aeca8b1-faa7-4a6d-a12f-7a31aebe9ffc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:22.062436433Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1397,"client":"2","":"downstairs","session_id":"0aeca8b1-faa7-4a6d-a12f-7a31aebe9ffc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:22.062446112Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1397,"session_id":"0aeca8b1-faa7-4a6d-a12f-7a31aebe9ffc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:22.062455162Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1397,"session_id":"0aeca8b1-faa7-4a6d-a12f-7a31aebe9ffc"} {"msg":"ea350bc0-2663-4c87-af1e-694f1ab59c4d is now active with session: 0aeca8b1-faa7-4a6d-a12f-7a31aebe9ffc","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:22.062464982Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1397,"session_id":"0aeca8b1-faa7-4a6d-a12f-7a31aebe9ffc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:22.062473602Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1397,"session_id":"0aeca8b1-faa7-4a6d-a12f-7a31aebe9ffc"} {"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:22.062482531Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1397} {"msg":"Activated sub_volume ea350bc0-2663-4c87-af1e-694f1ab59c4d","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:22.062490481Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1397} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:22.062498431Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1397,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:22.062827971Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1397,"task":"crutest"} 01/30 Write block 481 len 8192 data: 9 6 02/30 Read block 400 len 40960 03/30 Write block 376 len 40960 data: 8 8 9 10 9 8 8 6 7 7 04/30 Read block 543 len 36864 05/30 Flush 06/30 Write block 231 len 28672 data: 8 7 7 6 4 3 3 07/30 Read block 391 len 16384 08/30 Read block 439 len 36864 09/30 Read block 504 len 12288 10/30 Read block 205 len 28672 11/30 Read block 93 len 32768 12/30 Read block 216 len 20480 13/30 Write block 158 len 12288 data: 4 6 7 14/30 Read block 81 len 24576 15/30 Write block 255 len 36864 data: 8 6 4 4 4 4 4 4 3 16/30 Read block 104 len 20480 17/30 Write block 284 len 8192 data: 5 7 18/30 Read block 469 len 32768 19/30 Write block 353 len 16384 data: 4 5 4 5 20/30 Read block 218 len 8192 21/30 Read block 281 len 24576 22/30 Write block 113 len 12288 data: 7 6 5 23/30 Write block 227 len 4096 data: 10 24/30 Read block 224 len 4096 25/30 Write block 468 len 16384 data: 4 5 5 3 26/30 Write block 41 len 40960 data: 3 3 3 3 3 2 3 4 3 4 27/30 Read block 583 len 40960 28/30 Read block 405 len 8192 29/30 Read block 48 len 8192 30/30 Write block 309 len 20480 data: 6 5 5 6 6 ---------------------------------------------------------------- Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Read 10 Done Done Done false 1002 Acked Write 10 Sent Done Done false 1003 Acked Read 9 Sent Done Done false 1004 Acked Flush 0 Sent Done Done false 1005 Acked Write 7 Sent Done Done false 1006 Acked Read 4 Sent Done Done false 1007 Acked Read 9 Sent Done Done false 1008 Acked Read 3 Sent Done Done false 1009 Acked Read 7 Sent Done Done false 1010 Acked Read 8 Sent Done Done false 1011 Acked Read 5 Sent Done Done false 1012 Acked Write 3 Sent Done Done false 1013 Acked Read 6 Sent Done Done false 1014 Acked Write 9 Sent Done Done false 1015 Acked Read 5 Sent Done Done false 1016 Acked Write 2 Sent Done Done false 1017 Acked Read 8 Sent Done Done false 1018 Acked Write 4 Sent Done Done false 1019 Acked Read 2 Sent Done Done false 1020 Acked Read 6 Sent Done Done false 1021 Acked Write 3 Sent Done Done false 1022 Acked Write 1 Sent Done Done false 1023 Acked Read 1 Sent Done Done false 1024 Acked Write 4 Sent Done Done false 1025 Acked Write 10 Sent Done Done false 1026 Acked Read 10 Sent Done Done false 1027 Acked Read 2 Sent Done Done false 1028 Acked Read 2 Sent Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 1 1 30 Done 2 29 29 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1004)) Some(JobId(1004)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:44:22.071831752Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1397} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:22.071976357Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1397,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 61 61 61 99 99 99 F F F 1 020-039 58 58 58 95 95 95 F F F 2 040-059 64 64 64 106 106 106 F T T <--- 3 060-079 64 64 64 105 105 105 F F F 4 080-099 57 57 57 94 94 94 F F F 5 100-119 64 64 64 105 105 105 F T T <--- 6 120-139 64 64 64 105 105 105 F F F 7 140-159 63 63 63 103 103 103 F T T <--- 8 160-179 63 63 63 103 103 103 F T T <--- 9 180-199 64 64 64 104 104 104 F F F 10 200-219 64 64 64 104 104 104 F F F 11 220-239 64 64 64 105 105 105 F T T <--- 12 240-259 63 63 63 103 103 103 F T T <--- 13 260-279 64 64 64 105 105 105 F T T <--- 14 280-299 60 60 60 97 97 97 F T T <--- 15 300-319 61 61 61 99 99 99 F T T <--- 16 320-339 65 65 65 107 107 107 F F F 17 340-359 62 62 62 102 102 102 F T T <--- 18 360-379 62 66 66 102 108 108 F F F <--- 19 380-399 63 66 66 103 108 108 F F F <--- 20 400-419 61 61 61 99 99 99 F F F 21 420-439 64 64 64 105 105 105 F F F 22 440-459 58 58 58 95 95 95 F F F 23 460-479 65 65 65 107 107 107 F T T <--- 24 480-499 64 66 66 105 108 108 T F F <--- 25 500-519 58 58 58 95 95 95 F F F 26 520-539 60 60 60 98 98 98 F F F 27 540-559 62 62 62 100 100 100 F F F 28 560-579 65 65 65 107 107 107 F F F 29 580-599 64 64 64 105 105 105 F F F Max gen: 66, Max flush: 108 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 33 Verifying data now No change after verify Loop: 33 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 61 61 61 99 99 99 F F F 1 020-039 58 58 58 95 95 95 F F F 2 040-059 67 67 67 109 109 109 F F F 3 060-079 64 64 64 105 105 105 F F F 4 080-099 57 57 57 94 94 94 F F F 5 100-119 67 67 67 109 109 109 F F F 6 120-139 64 64 64 105 105 105 F F F 7 140-159 67 67 67 109 109 109 F F F 8 160-179 67 67 67 109 109 109 F F F 9 180-199 64 64 64 104 104 104 F F F 10 200-219 64 64 64 104 104 104 F F F 11 220-239 67 67 67 109 109 109 F F F 12 240-259 67 67 67 109 109 109 F F F 13 260-279 67 67 67 109 109 109 F F F 14 280-299 67 67 67 109 109 109 F F F 15 300-319 67 67 67 109 109 109 F F F 16 320-339 65 65 65 107 107 107 F F F 17 340-359 67 67 67 109 109 109 F F F 18 360-379 66 66 66 108 108 108 F F F 19 380-399 66 66 66 108 108 108 F F F 20 400-419 61 61 61 99 99 99 F F F 21 420-439 64 64 64 105 105 105 F F F 22 440-459 58 58 58 95 95 95 F F F 23 460-479 67 67 67 109 109 109 F F F 24 480-499 66 66 66 108 108 108 F F F 25 500-519 58 58 58 95 95 95 F F F 26 520-539 60 60 60 98 98 98 F F F 27 540-559 62 62 62 100 100 100 F F F 28 560-579 65 65 65 107 107 107 F F F 29 580-599 64 64 64 105 105 105 F F F Max gen: 67, Max flush: 109 Begin loop 34 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:26.328864599Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1405,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:26.336912877Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1405,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:26.337199579Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1405} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:26.337223808Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1405} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:26.337232708Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1405} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:26.337244107Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1405} {"msg":"Crucible eca68102-615b-4fcf-b062-0ea513542f46 has session id: 72b76366-df14-40ba-b726-fae0ab32d979","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:26.337578988Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1405,"session_id":"72b76366-df14-40ba-b726-fae0ab32d979"} {"msg":"Upstairs opts: Upstairs UUID: eca68102-615b-4fcf-b062-0ea513542f46, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:26.337617177Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1405,"session_id":"72b76366-df14-40ba-b726-fae0ab32d979"} {"msg":"Crucible stats registered with UUID: eca68102-615b-4fcf-b062-0ea513542f46","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:26.337627146Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1405,"session_id":"72b76366-df14-40ba-b726-fae0ab32d979"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:44:26.337636096Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1405,"session_id":"72b76366-df14-40ba-b726-fae0ab32d979"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:26.337789422Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1405,"":"io task","client":"2","":"downstairs","session_id":"72b76366-df14-40ba-b726-fae0ab32d979"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:26.337821471Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1405,"":"io task","client":"0","":"downstairs","session_id":"72b76366-df14-40ba-b726-fae0ab32d979"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:26.33783254Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1405,"":"io task","client":"1","":"downstairs","session_id":"72b76366-df14-40ba-b726-fae0ab32d979"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:26.337974966Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1405,"task":"crutest"} {"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:26.337993016Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1405} {"msg":"eca68102-615b-4fcf-b062-0ea513542f46 active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:26.338011845Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1405,"session_id":"72b76366-df14-40ba-b726-fae0ab32d979"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:26.338020605Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1405,"client":"0","":"downstairs","session_id":"72b76366-df14-40ba-b726-fae0ab32d979"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:26.338030275Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1405,"client":"1","":"downstairs","session_id":"72b76366-df14-40ba-b726-fae0ab32d979"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:26.338039674Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1405,"client":"2","":"downstairs","session_id":"72b76366-df14-40ba-b726-fae0ab32d979"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:26.338049184Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1405,"":"io task","client":"2","":"downstairs","session_id":"72b76366-df14-40ba-b726-fae0ab32d979"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:26.338059184Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1405,"":"io task","client":"0","":"downstairs","session_id":"72b76366-df14-40ba-b726-fae0ab32d979"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:26.338071134Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1405,"":"io task","client":"1","":"downstairs","session_id":"72b76366-df14-40ba-b726-fae0ab32d979"} {"msg":"ds_connection connected from Ok(127.0.0.1:56578)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:26.338141652Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1405,"":"io task","client":"2","":"downstairs","session_id":"72b76366-df14-40ba-b726-fae0ab32d979"} {"msg":"ds_connection connected from Ok(127.0.0.1:47028)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:26.338166981Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1405,"":"io task","client":"1","":"downstairs","session_id":"72b76366-df14-40ba-b726-fae0ab32d979"} {"msg":"ds_connection connected from Ok(127.0.0.1:50792)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:26.3381782Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1405,"":"io task","client":"0","":"downstairs","session_id":"72b76366-df14-40ba-b726-fae0ab32d979"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:26.338824962Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1405,"client":"0","":"downstairs","session_id":"72b76366-df14-40ba-b726-fae0ab32d979"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:26.33887444Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1405,"client":"2","":"downstairs","session_id":"72b76366-df14-40ba-b726-fae0ab32d979"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:26.33889088Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1405,"client":"1","":"downstairs","session_id":"72b76366-df14-40ba-b726-fae0ab32d979"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:26.339142883Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1405,"session_id":"72b76366-df14-40ba-b726-fae0ab32d979"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:26.339178452Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1405,"session_id":"72b76366-df14-40ba-b726-fae0ab32d979"} {"msg":"[0]R flush_numbers[0..12]: [99, 95, 109, 105, 94, 109, 105, 109, 109, 104, 104, 109]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:26.33923821Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1405,"":"downstairs","":"downstairs","session_id":"72b76366-df14-40ba-b726-fae0ab32d979"} {"msg":"[0]R generation[0..12]: [61, 58, 67, 64, 57, 67, 64, 67, 67, 64, 64, 67]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:26.3392493Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1405,"":"downstairs","":"downstairs","session_id":"72b76366-df14-40ba-b726-fae0ab32d979"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:26.339260679Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1405,"":"downstairs","":"downstairs","session_id":"72b76366-df14-40ba-b726-fae0ab32d979"} {"msg":"[1]R flush_numbers[0..12]: [99, 95, 109, 105, 94, 109, 105, 109, 109, 104, 104, 109]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:26.339271259Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1405,"":"downstairs","":"downstairs","session_id":"72b76366-df14-40ba-b726-fae0ab32d979"} {"msg":"[1]R generation[0..12]: [61, 58, 67, 64, 57, 67, 64, 67, 67, 64, 64, 67]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:26.339281479Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1405,"":"downstairs","":"downstairs","session_id":"72b76366-df14-40ba-b726-fae0ab32d979"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:26.339292308Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1405,"":"downstairs","":"downstairs","session_id":"72b76366-df14-40ba-b726-fae0ab32d979"} {"msg":"[2]R flush_numbers[0..12]: [99, 95, 109, 105, 94, 109, 105, 109, 109, 104, 104, 109]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:26.339305968Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1405,"":"downstairs","":"downstairs","session_id":"72b76366-df14-40ba-b726-fae0ab32d979"} {"msg":"[2]R generation[0..12]: [61, 58, 67, 64, 57, 67, 64, 67, 67, 64, 64, 67]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:26.339316168Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1405,"":"downstairs","":"downstairs","session_id":"72b76366-df14-40ba-b726-fae0ab32d979"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:26.339326787Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1405,"":"downstairs","":"downstairs","session_id":"72b76366-df14-40ba-b726-fae0ab32d979"} {"msg":"Max found gen is 68","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:26.339336157Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1405,"":"downstairs","":"downstairs","session_id":"72b76366-df14-40ba-b726-fae0ab32d979"} {"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:26.339345757Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1405,"":"downstairs","":"downstairs","session_id":"72b76366-df14-40ba-b726-fae0ab32d979"} {"msg":"Next flush: 110","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:26.339355027Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1405,"":"downstairs","":"downstairs","session_id":"72b76366-df14-40ba-b726-fae0ab32d979"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:26.339364306Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1405,"":"downstairs","":"downstairs","session_id":"72b76366-df14-40ba-b726-fae0ab32d979"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:26.339373836Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1405,"session_id":"72b76366-df14-40ba-b726-fae0ab32d979"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:26.339382506Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1405,"client":"0","":"downstairs","session_id":"72b76366-df14-40ba-b726-fae0ab32d979"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:26.339392175Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1405,"client":"1","":"downstairs","session_id":"72b76366-df14-40ba-b726-fae0ab32d979"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:26.339401565Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1405,"client":"2","":"downstairs","session_id":"72b76366-df14-40ba-b726-fae0ab32d979"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:26.339411275Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1405,"session_id":"72b76366-df14-40ba-b726-fae0ab32d979"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:26.339420365Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1405,"session_id":"72b76366-df14-40ba-b726-fae0ab32d979"} {"msg":"eca68102-615b-4fcf-b062-0ea513542f46 is now active with session: 72b76366-df14-40ba-b726-fae0ab32d979","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:26.339430334Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1405,"session_id":"72b76366-df14-40ba-b726-fae0ab32d979"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:26.339439024Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1405,"session_id":"72b76366-df14-40ba-b726-fae0ab32d979"} {"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:26.339449614Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1405} {"msg":"Activated sub_volume eca68102-615b-4fcf-b062-0ea513542f46","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:26.339457474Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1405} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:26.339465493Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1405,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:26.339775734Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1405,"task":"crutest"} 01/30 Write block 234 len 4096 data: 7 02/30 Write block 220 len 24576 data: 5 5 7 9 10 11 03/30 Write block 396 len 28672 data: 4 4 4 5 5 5 5 04/30 Write block 410 len 40960 data: 6 6 6 6 4 3 3 3 4 5 05/30 Read block 378 len 40960 06/30 Write block 230 len 8192 data: 8 9 07/30 Read block 343 len 4096 08/30 Read block 458 len 28672 09/30 Read block 428 len 28672 10/30 Read block 431 len 20480 11/30 Read block 64 len 12288 12/30 Read block 240 len 12288 13/30 Write block 63 len 20480 data: 5 4 5 5 5 14/30 Write block 323 len 20480 data: 7 9 8 7 7 15/30 Write block 450 len 32768 data: 4 4 3 4 3 2 4 5 16/30 Read block 7 len 24576 17/30 Write block 157 len 12288 data: 3 5 7 18/30 Write block 586 len 8192 data: 5 6 19/30 Write block 159 len 32768 data: 8 8 4 4 5 4 7 7 20/30 Write block 215 len 32768 data: 7 8 9 7 6 6 6 8 21/30 Write block 519 len 36864 data: 7 7 8 8 9 8 7 6 6 22/30 Read block 527 len 4096 23/30 Write block 188 len 28672 data: 6 6 5 5 5 4 5 24/30 Write block 199 len 24576 data: 5 3 8 9 8 9 25/30 Write block 416 len 28672 data: 4 4 5 6 6 6 6 26/30 Read block 232 len 12288 27/30 Read block 71 len 40960 28/30 Read block 183 len 16384 29/30 Read block 110 len 32768 30/30 Write block 381 len 28672 data: 9 9 7 8 8 6 6 ---------------------------------------------------------------- Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Write 6 Done Done Done false 1002 Acked Write 7 Done Done Done false 1003 Acked Write 10 Done Done Done false 1004 Acked Read 10 Done Done Done false 1005 Acked Write 2 Done Sent Done false 1006 Acked Read 1 Done Sent Done false 1007 Acked Read 7 Done Sent Done false 1008 Acked Read 7 Done Sent Done false 1009 Acked Read 5 Done Sent Done false 1010 Acked Read 3 Done Sent Done false 1011 Acked Read 3 Done Sent Done false 1012 Acked Write 5 Done Sent Done false 1013 Acked Write 5 Done Sent Done false 1014 Acked Write 8 Done Sent Done false 1015 Acked Read 6 Done Sent Done false 1016 Acked Write 3 Done Sent Done false 1017 Acked Write 2 Done Sent Done false 1018 Acked Write 8 Done Sent Done false 1019 Acked Write 8 Done Sent Done false 1020 Acked Write 9 Done Sent Done false 1021 Acked Read 1 Done Sent Done false 1022 Acked Write 7 Done Sent Done false 1023 Acked Write 6 Done Sent Done false 1024 Acked Write 7 Done Sent Done false 1025 Acked Read 3 Done Sent Done false 1026 Acked Read 10 Done Sent Done false 1027 Acked Read 4 Done Sent Done false 1028 Acked Read 8 Done Sent Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 25 1 27 Done 29 5 29 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:44:26.347096883Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1405} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:26.347302137Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1405,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 61 61 61 99 99 99 F F F 1 020-039 58 58 58 95 95 95 F F F 2 040-059 67 67 67 109 109 109 F F F 3 060-079 64 64 64 105 105 105 T F T <--- 4 080-099 57 57 57 94 94 94 F F F 5 100-119 67 67 67 109 109 109 F F F 6 120-139 64 64 64 105 105 105 F F F 7 140-159 67 67 67 109 109 109 T F T <--- 8 160-179 67 67 67 109 109 109 T F T <--- 9 180-199 64 64 64 104 104 104 T F T <--- 10 200-219 64 64 64 104 104 104 T F T <--- 11 220-239 67 67 67 109 109 109 T T T 12 240-259 67 67 67 109 109 109 F F F 13 260-279 67 67 67 109 109 109 F F F 14 280-299 67 67 67 109 109 109 F F F 15 300-319 67 67 67 109 109 109 F F F 16 320-339 65 65 65 107 107 107 T F T <--- 17 340-359 67 67 67 109 109 109 F F F 18 360-379 66 66 66 108 108 108 F F F 19 380-399 66 66 66 108 108 108 T T T 20 400-419 61 61 61 99 99 99 T T T 21 420-439 64 64 64 105 105 105 T F T <--- 22 440-459 58 58 58 95 95 95 T F T <--- 23 460-479 67 67 67 109 109 109 F F F 24 480-499 66 66 66 108 108 108 F F F 25 500-519 58 58 58 95 95 95 T F T <--- 26 520-539 60 60 60 98 98 98 T F T <--- 27 540-559 62 62 62 100 100 100 F F F 28 560-579 65 65 65 107 107 107 F F F 29 580-599 64 64 64 105 105 105 T F T <--- Max gen: 67, Max flush: 109 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 34 Verifying data now No change after verify Loop: 34 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 61 61 61 99 99 99 F F F 1 020-039 58 58 58 95 95 95 F F F 2 040-059 67 67 67 109 109 109 F F F 3 060-079 68 68 68 110 110 110 F F F 4 080-099 57 57 57 94 94 94 F F F 5 100-119 67 67 67 109 109 109 F F F 6 120-139 64 64 64 105 105 105 F F F 7 140-159 68 68 68 110 110 110 F F F 8 160-179 68 68 68 110 110 110 F F F 9 180-199 68 68 68 110 110 110 F F F 10 200-219 68 68 68 110 110 110 F F F 11 220-239 68 68 68 110 110 110 F F F 12 240-259 67 67 67 109 109 109 F F F 13 260-279 67 67 67 109 109 109 F F F 14 280-299 67 67 67 109 109 109 F F F 15 300-319 67 67 67 109 109 109 F F F 16 320-339 68 68 68 110 110 110 F F F 17 340-359 67 67 67 109 109 109 F F F 18 360-379 66 66 66 108 108 108 F F F 19 380-399 68 68 68 110 110 110 F F F 20 400-419 68 68 68 110 110 110 F F F 21 420-439 68 68 68 110 110 110 F F F 22 440-459 68 68 68 110 110 110 F F F 23 460-479 67 67 67 109 109 109 F F F 24 480-499 66 66 66 108 108 108 F F F 25 500-519 68 68 68 110 110 110 F F F 26 520-539 68 68 68 110 110 110 F F F 27 540-559 62 62 62 100 100 100 F F F 28 560-579 65 65 65 107 107 107 F F F 29 580-599 68 68 68 110 110 110 F F F Max gen: 68, Max flush: 110 Begin loop 35 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:30.596818672Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1413,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:30.604825731Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1413,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:30.605136942Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1413} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:30.605162051Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1413} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:30.605170871Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1413} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:30.60518233Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1413} {"msg":"Crucible 2d0edbce-064e-4d9d-b0a3-b09e9e6474f3 has session id: c5ce4cf3-aac1-4476-a4c3-fa90b7009b5b","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:30.60554617Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1413,"session_id":"c5ce4cf3-aac1-4476-a4c3-fa90b7009b5b"} {"msg":"Upstairs opts: Upstairs UUID: 2d0edbce-064e-4d9d-b0a3-b09e9e6474f3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:30.605574599Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1413,"session_id":"c5ce4cf3-aac1-4476-a4c3-fa90b7009b5b"} {"msg":"Crucible stats registered with UUID: 2d0edbce-064e-4d9d-b0a3-b09e9e6474f3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:30.605585839Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1413,"session_id":"c5ce4cf3-aac1-4476-a4c3-fa90b7009b5b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:44:30.605594968Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1413,"session_id":"c5ce4cf3-aac1-4476-a4c3-fa90b7009b5b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:30.605723275Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1413,"":"io task","client":"0","":"downstairs","session_id":"c5ce4cf3-aac1-4476-a4c3-fa90b7009b5b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:30.605757024Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1413,"":"io task","client":"2","":"downstairs","session_id":"c5ce4cf3-aac1-4476-a4c3-fa90b7009b5b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:30.605768113Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1413,"":"io task","client":"1","":"downstairs","session_id":"c5ce4cf3-aac1-4476-a4c3-fa90b7009b5b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:30.60587769Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1413,"task":"crutest"} {"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:30.60588997Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1413} {"msg":"2d0edbce-064e-4d9d-b0a3-b09e9e6474f3 active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:30.605921909Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1413,"session_id":"c5ce4cf3-aac1-4476-a4c3-fa90b7009b5b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:30.605931439Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1413,"client":"0","":"downstairs","session_id":"c5ce4cf3-aac1-4476-a4c3-fa90b7009b5b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:30.605941178Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1413,"client":"1","":"downstairs","session_id":"c5ce4cf3-aac1-4476-a4c3-fa90b7009b5b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:30.605950718Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1413,"client":"2","":"downstairs","session_id":"c5ce4cf3-aac1-4476-a4c3-fa90b7009b5b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:30.605960238Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1413,"":"io task","client":"2","":"downstairs","session_id":"c5ce4cf3-aac1-4476-a4c3-fa90b7009b5b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:30.605970238Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1413,"":"io task","client":"0","":"downstairs","session_id":"c5ce4cf3-aac1-4476-a4c3-fa90b7009b5b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:30.605980167Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1413,"":"io task","client":"1","":"downstairs","session_id":"c5ce4cf3-aac1-4476-a4c3-fa90b7009b5b"} {"msg":"ds_connection connected from Ok(127.0.0.1:54794)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:30.606039746Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1413,"":"io task","client":"0","":"downstairs","session_id":"c5ce4cf3-aac1-4476-a4c3-fa90b7009b5b"} {"msg":"ds_connection connected from Ok(127.0.0.1:60980)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:30.606064625Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1413,"":"io task","client":"1","":"downstairs","session_id":"c5ce4cf3-aac1-4476-a4c3-fa90b7009b5b"} {"msg":"ds_connection connected from Ok(127.0.0.1:53749)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:30.606075865Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1413,"":"io task","client":"2","":"downstairs","session_id":"c5ce4cf3-aac1-4476-a4c3-fa90b7009b5b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:30.606784424Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1413,"client":"2","":"downstairs","session_id":"c5ce4cf3-aac1-4476-a4c3-fa90b7009b5b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:30.606810543Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1413,"client":"1","":"downstairs","session_id":"c5ce4cf3-aac1-4476-a4c3-fa90b7009b5b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:30.606827973Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1413,"client":"0","":"downstairs","session_id":"c5ce4cf3-aac1-4476-a4c3-fa90b7009b5b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:30.607104375Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1413,"session_id":"c5ce4cf3-aac1-4476-a4c3-fa90b7009b5b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:30.607123014Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1413,"session_id":"c5ce4cf3-aac1-4476-a4c3-fa90b7009b5b"} {"msg":"[0]R flush_numbers[0..12]: [99, 95, 109, 110, 94, 109, 105, 110, 110, 110, 110, 110]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:30.607210092Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1413,"":"downstairs","":"downstairs","session_id":"c5ce4cf3-aac1-4476-a4c3-fa90b7009b5b"} {"msg":"[0]R generation[0..12]: [61, 58, 67, 68, 57, 67, 64, 68, 68, 68, 68, 68]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:30.607221152Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1413,"":"downstairs","":"downstairs","session_id":"c5ce4cf3-aac1-4476-a4c3-fa90b7009b5b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:30.607232141Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1413,"":"downstairs","":"downstairs","session_id":"c5ce4cf3-aac1-4476-a4c3-fa90b7009b5b"} {"msg":"[1]R flush_numbers[0..12]: [99, 95, 109, 110, 94, 109, 105, 110, 110, 110, 110, 110]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:30.607242761Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1413,"":"downstairs","":"downstairs","session_id":"c5ce4cf3-aac1-4476-a4c3-fa90b7009b5b"} {"msg":"[1]R generation[0..12]: [61, 58, 67, 68, 57, 67, 64, 68, 68, 68, 68, 68]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:30.607253221Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1413,"":"downstairs","":"downstairs","session_id":"c5ce4cf3-aac1-4476-a4c3-fa90b7009b5b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:30.60726402Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1413,"":"downstairs","":"downstairs","session_id":"c5ce4cf3-aac1-4476-a4c3-fa90b7009b5b"} {"msg":"[2]R flush_numbers[0..12]: [99, 95, 109, 110, 94, 109, 105, 110, 110, 110, 110, 110]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:30.60727477Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1413,"":"downstairs","":"downstairs","session_id":"c5ce4cf3-aac1-4476-a4c3-fa90b7009b5b"} {"msg":"[2]R generation[0..12]: [61, 58, 67, 68, 57, 67, 64, 68, 68, 68, 68, 68]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:30.60728503Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1413,"":"downstairs","":"downstairs","session_id":"c5ce4cf3-aac1-4476-a4c3-fa90b7009b5b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:30.607295889Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1413,"":"downstairs","":"downstairs","session_id":"c5ce4cf3-aac1-4476-a4c3-fa90b7009b5b"} {"msg":"Max found gen is 69","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:30.607305449Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1413,"":"downstairs","":"downstairs","session_id":"c5ce4cf3-aac1-4476-a4c3-fa90b7009b5b"} {"msg":"Generation requested: 70 >= found:69","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:30.607315169Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1413,"":"downstairs","":"downstairs","session_id":"c5ce4cf3-aac1-4476-a4c3-fa90b7009b5b"} {"msg":"Next flush: 111","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:30.607325939Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1413,"":"downstairs","":"downstairs","session_id":"c5ce4cf3-aac1-4476-a4c3-fa90b7009b5b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:30.607335298Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1413,"":"downstairs","":"downstairs","session_id":"c5ce4cf3-aac1-4476-a4c3-fa90b7009b5b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:30.607345068Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1413,"session_id":"c5ce4cf3-aac1-4476-a4c3-fa90b7009b5b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:30.607353808Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1413,"client":"0","":"downstairs","session_id":"c5ce4cf3-aac1-4476-a4c3-fa90b7009b5b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:30.607368457Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1413,"client":"1","":"downstairs","session_id":"c5ce4cf3-aac1-4476-a4c3-fa90b7009b5b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:30.607378077Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1413,"client":"2","":"downstairs","session_id":"c5ce4cf3-aac1-4476-a4c3-fa90b7009b5b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:30.607388077Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1413,"session_id":"c5ce4cf3-aac1-4476-a4c3-fa90b7009b5b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:30.607397136Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1413,"session_id":"c5ce4cf3-aac1-4476-a4c3-fa90b7009b5b"} {"msg":"2d0edbce-064e-4d9d-b0a3-b09e9e6474f3 is now active with session: c5ce4cf3-aac1-4476-a4c3-fa90b7009b5b","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:30.607407016Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1413,"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 session_id":"c5ce4cf3-aac1-4476-a4c3-fa90b7009b5b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:30.607420966Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1413,"session_id":"c5ce4cf3-aac1-4476-a4c3-fa90b7009b5b"} {"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:30.607429976Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1413} {"msg":"Activated sub_volume 2d0edbce-064e-4d9d-b0a3-b09e9e6474f3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:30.607437925Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1413} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:30.607445975Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1413,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:30.607670439Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1413,"task":"crutest"} 01/30 Write block 530 len 16384 data: 4 4 9 7 02/30 Read block 554 len 12288 03/30 Read block 445 len 28672 04/30 Read block 172 len 20480 05/30 Write block 321 len 4096 data: 5 06/30 Read block 431 len 20480 07/30 Read block 201 len 24576 08/30 Read block 149 len 32768 09/30 Write block 208 len 16384 data: 7 7 8 7 10/30 Write block 203 len 32768 data: 9 10 8 8 6 8 8 9 11/30 Write block 488 len 20480 data: 6 5 6 7 5 12/30 Read block 84 len 4096 13/30 Write block 41 len 8192 data: 4 4 14/30 Write block 393 len 20480 data: 4 3 4 5 5 15/30 Read block 427 len 20480 16/30 Read block 299 len 24576 17/30 Read block 245 len 16384 18/30 Read block 205 len 8192 19/30 Read block 484 len 12288 20/30 Write block 578 len 20480 data: 7 5 3 4 6 21/30 Write block 300 len 16384 data: 7 7 4 4 22/30 Write block 137 len 8192 data: 8 8 23/30 Read block 219 len 4096 24/30 Write block 329 len 36864 data: 6 7 7 6 6 7 8 9 10 25/30 Read block 589 len 4096 26/30 Write block 184 len 16384 data: 6 5 6 7 27/30 Read block 186 len 12288 28/30 Write block 376 len 28672 data: 9 9 10 11 10 10 10 29/30 Read block 383 len 28672 30/30 Write block 166 len 28672 data: 8 7 8 9 9 8 7 ---------------------------------------------------------------- Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Sent Done false 1001 Acked Read 3 Done Sent Done false 1002 Acked Read 7 Done Sent Done false 1003 Acked Read 5 Done Sent Done false 1004 Acked Write 1 Done Sent Done false 1005 Acked Read 5 Done Sent Done false 1006 Acked Read 6 Done Sent Done false 1007 Acked Read 8 Done Sent Done false 1008 Acked Write 4 Done Sent Done false 1009 Acked Write 8 Done Sent Done false 1010 Acked Write 5 Done Sent Done false 1011 Acked Read 1 Done Sent Done false 1012 Acked Write 2 Done Sent Done false 1013 Acked Write 5 Done Sent Done false 1014 Acked Read 5 Done Sent Done false 1015 Acked Read 6 Done Sent Done false 1016 Acked Read 4 Done Sent Done false 1017 Acked Read 2 Done Sent Done false 1018 Acked Read 3 Done Sent Done false 1019 Acked Write 5 Done Sent Done false 1020 Acked Write 4 Done Sent Done false 1021 Acked Write 2 Done Sent Done false 1022 Acked Read 1 Done Sent Done false 1023 Acked Write 9 Done Sent Done false 1024 Acked Read 1 Done Sent Done false 1025 Acked Write 4 Done Sent Done false 1026 Acked Read 3 Done Sent Done false 1027 Acked Write 7 Done Sent Done false 1028 Acked Read 7 Done Sent Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 30 1 32 Done 29 0 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:44:30.615331608Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1413} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:30.615548791Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1413,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 61 61 61 99 99 99 F F F 1 020-039 58 58 58 95 95 95 F F F 2 040-059 67 67 67 109 109 109 T F T <--- 3 060-079 68 68 68 110 110 110 F F F 4 080-099 57 57 57 94 94 94 F F F 5 100-119 67 67 67 109 109 109 F F F 6 120-139 64 64 64 105 105 105 T F T <--- 7 140-159 68 68 68 110 110 110 F F F 8 160-179 68 68 68 110 110 110 T F T <--- 9 180-199 68 68 68 110 110 110 T F T <--- 10 200-219 68 68 68 110 110 110 T F T <--- 11 220-239 68 68 68 110 110 110 F F F 12 240-259 67 67 67 109 109 109 F F F 13 260-279 67 67 67 109 109 109 F F F 14 280-299 67 67 67 109 109 109 F F F 15 300-319 67 67 67 109 109 109 T F T <--- 16 320-339 68 68 68 110 110 110 T F T <--- 17 340-359 67 67 67 109 109 109 F F F 18 360-379 66 66 66 108 108 108 T F T <--- 19 380-399 68 68 68 110 110 110 T F T <--- 20 400-419 68 68 68 110 110 110 F F F 21 420-439 68 68 68 110 110 110 F F F 22 440-459 68 68 68 110 110 110 F F F 23 460-479 67 67 67 109 109 109 F F F 24 480-499 66 66 66 108 108 108 T F T <--- 25 500-519 68 68 68 110 110 110 F F F 26 520-539 68 68 68 110 110 110 T F T <--- 27 540-559 62 62 62 100 100 100 F F F 28 560-579 65 65 65 107 107 107 T F T <--- 29 580-599 68 68 68 110 110 110 T F T <--- Max gen: 68, Max flush: 110 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 35 Verifying data now No change after verify Loop: 35 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 61 61 61 99 99 99 F F F 1 020-039 58 58 58 95 95 95 F F F 2 040-059 69 69 69 111 111 111 F F F 3 060-079 68 68 68 110 110 110 F F F 4 080-099 57 57 57 94 94 94 F F F 5 100-119 67 67 67 109 109 109 F F F 6 120-139 69 69 69 111 111 111 F F F 7 140-159 68 68 68 110 110 110 F F F 8 160-179 69 69 69 111 111 111 F F F 9 180-199 69 69 69 111 111 111 F F F 10 200-219 69 69 69 111 111 111 F F F 11 220-239 68 68 68 110 110 110 F F F 12 240-259 67 67 67 109 109 109 F F F 13 260-279 67 67 67 109 109 109 F F F 14 280-299 67 67 67 109 109 109 F F F 15 300-319 69 69 69 111 111 111 F F F 16 320-339 69 69 69 111 111 111 F F F 17 340-359 67 67 67 109 109 109 F F F 18 360-379 69 69 69 111 111 111 F F F 19 380-399 69 69 69 111 111 111 F F F 20 400-419 68 68 68 110 110 110 F F F 21 420-439 68 68 68 110 110 110 F F F 22 440-459 68 68 68 110 110 110 F F F 23 460-479 67 67 67 109 109 109 F F F 24 480-499 69 69 69 111 111 111 F F F 25 500-519 68 68 68 110 110 110 F F F 26 520-539 69 69 69 111 111 111 F F F 27 540-559 62 62 62 100 100 100 F F F 28 560-579 69 69 69 111 111 111 F F F 29 580-599 69 69 69 111 111 111 F F F Max gen: 69, Max flush: 111 Begin loop 36 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:34.865214241Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1421,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:34.87320352Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1421,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:34.873488952Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1421} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:34.873514841Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1421} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:34.873525761Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1421} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:34.873537231Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1421} {"msg":"Crucible c8ce9b87-2d31-4715-927f-d8289f66674b has session id: fcc13046-7f0a-40b6-9a57-8c3e01011acf","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:34.873874701Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1421,"session_id":"fcc13046-7f0a-40b6-9a57-8c3e01011acf"} {"msg":"Upstairs opts: Upstairs UUID: c8ce9b87-2d31-4715-927f-d8289f66674b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:34.87389672Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1421,"session_id":"fcc13046-7f0a-40b6-9a57-8c3e01011acf"} {"msg":"Crucible stats registered with UUID: c8ce9b87-2d31-4715-927f-d8289f66674b","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:34.87390651Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1421,"session_id":"fcc13046-7f0a-40b6-9a57-8c3e01011acf"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:44:34.87391533Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1421,"session_id":"fcc13046-7f0a-40b6-9a57-8c3e01011acf"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:34.874061686Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1421,"":"io task","client":"0","":"downstairs","session_id":"fcc13046-7f0a-40b6-9a57-8c3e01011acf"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:34.874078915Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1421,"":"io task","client":"1","":"downstairs","session_id":"fcc13046-7f0a-40b6-9a57-8c3e01011acf"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:34.874088965Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1421,"":"io task","client":"2","":"downstairs","session_id":"fcc13046-7f0a-40b6-9a57-8c3e01011acf"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:34.8742571Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1421,"task":"crutest"} {"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:34.87427127Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1421} {"msg":"c8ce9b87-2d31-4715-927f-d8289f66674b active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:34.874304539Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1421,"session_id":"fcc13046-7f0a-40b6-9a57-8c3e01011acf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:34.874313988Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1421,"client":"0","":"downstairs","session_id":"fcc13046-7f0a-40b6-9a57-8c3e01011acf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:34.874323608Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1421,"client":"1","":"downstairs","session_id":"fcc13046-7f0a-40b6-9a57-8c3e01011acf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:34.874335158Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1421,"client":"2","":"downstairs","session_id":"fcc13046-7f0a-40b6-9a57-8c3e01011acf"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:34.874344677Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1421,"":"io task","client":"2","":"downstairs","session_id":"fcc13046-7f0a-40b6-9a57-8c3e01011acf"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:34.874354727Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1421,"":"io task","client":"0","":"downstairs","session_id":"fcc13046-7f0a-40b6-9a57-8c3e01011acf"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:34.874364567Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1421,"":"io task","client":"1","":"downstairs","session_id":"fcc13046-7f0a-40b6-9a57-8c3e01011acf"} {"msg":"ds_connection connected from Ok(127.0.0.1:60140)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:34.874466764Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1421,"":"io task","client":"0","":"downstairs","session_id":"fcc13046-7f0a-40b6-9a57-8c3e01011acf"} {"msg":"ds_connection connected from Ok(127.0.0.1:46515)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:34.874572881Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1421,"":"io task","client":"1","":"downstairs","session_id":"fcc13046-7f0a-40b6-9a57-8c3e01011acf"} {"msg":"ds_connection connected from Ok(127.0.0.1:52078)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:34.874584321Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1421,"":"io task","client":"2","":"downstairs","session_id":"fcc13046-7f0a-40b6-9a57-8c3e01011acf"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:34.875107085Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1421,"client":"1","":"downstairs","session_id":"fcc13046-7f0a-40b6-9a57-8c3e01011acf"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:34.875138065Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1421,"client":"0","":"downstairs","session_id":"fcc13046-7f0a-40b6-9a57-8c3e01011acf"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:34.875166824Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1421,"client":"2","":"downstairs","session_id":"fcc13046-7f0a-40b6-9a57-8c3e01011acf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:34.875419246Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1421,"session_id":"fcc13046-7f0a-40b6-9a57-8c3e01011acf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:34.875456115Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1421,"session_id":"fcc13046-7f0a-40b6-9a57-8c3e01011acf"} {"msg":"[0]R flush_numbers[0..12]: [99, 95, 111, 110, 94, 109, 111, 110, 111, 111, 111, 110]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:34.875494034Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1421,"":"downstairs","":"downstairs","session_id":"fcc13046-7f0a-40b6-9a57-8c3e01011acf"} {"msg":"[0]R generation[0..12]: [61, 58, 69, 68, 57, 67, 69, 68, 69, 69, 69, 68]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:34.875505004Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1421,"":"downstairs","":"downstairs","session_id":"fcc13046-7f0a-40b6-9a57-8c3e01011acf"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:34.875516014Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1421,"":"downstairs","":"downstairs","session_id":"fcc13046-7f0a-40b6-9a57-8c3e01011acf"} {"msg":"[1]R flush_numbers[0..12]: [99, 95, 111, 110, 94, 109, 111, 110, 111, 111, 111, 110]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:34.875528803Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1421,"":"downstairs","":"downstairs","session_id":"fcc13046-7f0a-40b6-9a57-8c3e01011acf"} {"msg":"[1]R generation[0..12]: [61, 58, 69, 68, 57, 67, 69, 68, 69, 69, 69, 68]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:34.875539183Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1421,"":"downstairs","":"downstairs","session_id":"fcc13046-7f0a-40b6-9a57-8c3e01011acf"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:34.875549963Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1421,"":"downstairs","":"downstairs","session_id":"fcc13046-7f0a-40b6-9a57-8c3e01011acf"} {"msg":"[2]R flush_numbers[0..12]: [99, 95, 111, 110, 94, 109, 111, 110, 111, 111, 111, 110]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:34.875560442Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1421,"":"downstairs","":"downstairs","session_id":"fcc13046-7f0a-40b6-9a57-8c3e01011acf"} {"msg":"[2]R generation[0..12]: [61, 58, 69, 68, 57, 67, 69, 68, 69, 69, 69, 68]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:34.875570662Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1421,"":"downstairs","":"downstairs","session_id":"fcc13046-7f0a-40b6-9a57-8c3e01011acf"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:34.875581552Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1421,"":"downstairs","":"downstairs","session_id":"fcc13046-7f0a-40b6-9a57-8c3e01011acf"} {"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:34.875590892Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1421,"":"downstairs","":"downstairs","session_id":"fcc13046-7f0a-40b6-9a57-8c3e01011acf"} {"msg":"Generation requested: 72 >= found:70","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:34.875600501Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1421,"":"downstairs","":"downstairs","session_id":"fcc13046-7f0a-40b6-9a57-8c3e01011acf"} {"msg":"Next flush: 112","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:34.875609791Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1421,"":"downstairs","":"downstairs","session_id":"fcc13046-7f0a-40b6-9a57-8c3e01011acf"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:34.875619071Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1421,"":"downstairs","":"downstairs","session_id":"fcc13046-7f0a-40b6-9a57-8c3e01011acf"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:34.87562868Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1421,"session_id":"fcc13046-7f0a-40b6-9a57-8c3e01011acf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:34.87563745Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1421,"client":"0","":"downstairs","session_id":"fcc13046-7f0a-40b6-9a57-8c3e01011acf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:34.87564707Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1421,"client":"1","":"downstairs","session_id":"fcc13046-7f0a-40b6-9a57-8c3e01011acf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:34.87565654Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1421,"client":"2","":"downstairs","session_id":"fcc13046-7f0a-40b6-9a57-8c3e01011acf"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:34.875667889Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1421,"session_id":"fcc13046-7f0a-40b6-9a57-8c3e01011acf"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:34.875676969Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1421,"session_id":"fcc13046-7f0a-40b6-9a57-8c3e01011acf"} {"msg":"c8ce9b87-2d31-4715-927f-d8289f66674b is now active with session: fcc13046-7f0a-40b6-9a57-8c3e01011acf","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:34.875686869Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1421,"session_id":"fcc13046-7f0a-40b6-9a57-8c3e01011acf"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:34.875695509Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1421,"session_id":"fcc13046-7f0a-40b6-9a57-8c3e01011acf"} {"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:34.875704568Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1421} {"msg":"Activated sub_volume c8ce9b87-2d31-4715-927f-d8289f66674b","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:34.875712748Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1421} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:34.875720888Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1421,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:34.876049638Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1421,"task":"crutest"} 01/30 Write block 538 len 16384 data: 10 10 10 7 02/30 Write block 0 len 40960 data: 3 3 5 4 6 8 10 8 8 6 03/30 Read block 493 len 36864 04/30 Read block 5 len 28672 05/30 Flush 06/30 Write block 117 len 36864 data: 6 5 7 7 7 7 7 6 5 07/30 Read block 451 len 36864 08/30 Flush 09/30 Write block 250 len 16384 data: 8 5 5 7 10/30 Write block 288 len 32768 data: 4 5 5 5 6 7 8 7 11/30 Write block 135 len 8192 data: 9 11 12/30 Read block 363 len 40960 13/30 Write block 326 len 36864 data: 8 8 6 7 8 8 7 7 8 14/30 Flush 15/30 Write block 580 len 36864 data: 4 5 7 6 6 6 6 7 7 16/30 Write block 326 len 16384 data: 9 9 7 8 17/30 Write block 408 len 28672 data: 5 5 7 7 7 7 5 18/30 Write block 438 len 8192 data: 5 5 19/30 Read block 514 len 24576 20/30 Read block 117 len 16384 21/30 Read block 35 len 40960 22/30 Read block 266 len 4096 23/30 Flush 24/30 Write block 238 len 36864 data: 3 5 5 5 4 3 4 4 6 25/30 Write block 194 len 32768 data: 6 4 4 4 5 6 4 9 26/30 Write block 30 len 16384 data: 4 6 6 6 27/30 Read block 529 len 40960 28/30 Write block 486 len 12288 data: 7 7 7 29/30 Read block 590 len 40960 30/30 Write block 218 len 12288 data: 8 7 7 ---------------------------------------------------------------- Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Write 10 Done Done Sent false 1002 Acked Read 9 Done Done Sent false 1003 Acked Read 7 Done Done Sent false 1004 Acked Flush 0 Done Done Sent false 1005 Acked Write 9 Done Done Sent false 1006 Acked Read 9 Done Done Sent false 1007 Acked Flush 0 Done Done Sent false 1008 Acked Write 4 Done Done Sent false 1009 Acked Write 8 Done Done Sent false 1010 Acked Write 2 Done Done Sent false 1011 Acked Read 10 Done Done Sent false 1012 Acked Write 9 Done Done Sent false 1013 Acked Flush 0 Done Done Sent false 1014 Acked Write 9 Done Done Sent false 1015 Acked Write 4 Done Done Sent false 1016 Acked Write 7 Done Done Sent false 1017 Acked Write 2 Done Done Sent false 1018 Acked Read 6 Done Done Sent false 1019 Acked Read 4 Done Done Sent false 1020 Acked Read 10 Done Done Sent false 1021 Acked Read 1 Done Done Sent false 1022 Acked Flush 0 Done Done Sent false 1023 Acked Write 9 Done Done Sent false 1024 Acked Write 8 Done Done Sent false 1025 Acked Write 4 Done Done Sent false 1026 Acked Read 10 Done Done Sent false 1027 Acked Write 3 Done Done Sent false 1028 Acked Read 10 Done Done Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 29 31 Done 29 29 1 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:44:34.885316881Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1421} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:34.885489586Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1421,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 72 72 61 112 112 99 F F F <--- 1 020-039 58 58 58 95 95 95 T T F <--- 2 040-059 69 69 69 111 111 111 F F F 3 060-079 68 68 68 110 110 110 F F F 4 080-099 57 57 57 94 94 94 F F F 5 100-119 72 72 67 113 113 109 F F F <--- 6 120-139 72 72 69 114 114 111 F F F <--- 7 140-159 68 68 68 110 110 110 F F F 8 160-179 69 69 69 111 111 111 F F F 9 180-199 69 69 69 111 111 111 T T F <--- 10 200-219 69 69 69 111 111 111 T T F <--- 11 220-239 68 68 68 110 110 110 T T F <--- 12 240-259 72 72 67 114 114 109 T T F <--- 13 260-279 67 67 67 109 109 109 F F F 14 280-299 72 72 67 114 114 109 F F F <--- 15 300-319 69 69 69 111 111 111 F F F 16 320-339 72 72 69 115 115 111 F F F <--- 17 340-359 67 67 67 109 109 109 F F F 18 360-379 69 69 69 111 111 111 F F F 19 380-399 69 69 69 111 111 111 F F F 20 400-419 72 72 68 115 115 110 F F F <--- 21 420-439 72 72 68 115 115 110 F F F <--- 22 440-459 68 68 68 110 110 110 F F F 23 460-479 67 67 67 109 109 109 F F F 24 480-499 69 69 69 111 111 111 T T F <--- 25 500-519 68 68 68 110 110 110 F F F 26 520-539 72 72 69 112 112 111 F F T <--- 27 540-559 72 72 62 112 112 100 F F T <--- 28 560-579 69 69 69 111 111 111 F F F 29 580-599 72 72 69 115 115 111 F F F <--- Max gen: 72, Max flush: 115 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 36 Verifying data now No change after verify Loop: 36 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 72 72 72 112 112 112 F F F 1 020-039 73 73 73 116 116 116 F F F 2 040-059 69 69 69 111 111 111 F F F 3 060-079 68 68 68 110 110 110 F F F 4 080-099 57 57 57 94 94 94 F F F 5 100-119 72 72 72 113 113 113 F F F 6 120-139 72 72 72 114 114 114 F F F 7 140-159 68 68 68 110 110 110 F F F 8 160-179 69 69 69 111 111 111 F F F 9 180-199 73 73 73 116 116 116 F F F 10 200-219 73 73 73 116 116 116 F F F 11 220-239 73 73 73 116 116 116 F F F 12 240-259 73 73 73 116 116 116 F F F 13 260-279 67 67 67 109 109 109 F F F 14 280-299 72 72 72 114 114 114 F F F 15 300-319 69 69 69 111 111 111 F F F 16 320-339 72 72 72 115 115 115 F F F 17 340-359 67 67 67 109 109 109 F F F 18 360-379 69 69 69 111 111 111 F F F 19 380-399 69 69 69 111 111 111 F F F 20 400-419 72 72 72 115 115 115 F F F 21 420-439 72 72 72 115 115 115 F F F 22 440-459 68 68 68 110 110 110 F F F 23 460-479 67 67 67 109 109 109 F F F 24 480-499 73 73 73 116 116 116 F F F 25 500-519 68 68 68 110 110 110 F F F 26 520-539 72 72 72 112 112 112 F F F 27 540-559 72 72 72 112 112 112 F F F 28 560-579 69 69 69 111 111 111 F F F 29 580-599 72 72 72 115 115 115 F F F Max gen: 73, Max flush: 116 Begin loop 37 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:39.145917906Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1429,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:44:39.152916104Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1429,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:39.152971332Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1429,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:39.159768946Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1429,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:39.160028728Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1429} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:39.160053438Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1429} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:39.160061858Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1429} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:39.160073477Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1429} {"msg":"Crucible 8b6ef580-5ae2-40fa-a8de-32e9e005b515 has session id: 32108433-215b-4274-9965-4e63faa6859a","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:39.160388098Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1429,"session_id":"32108433-215b-4274-9965-4e63faa6859a"} {"msg":"Upstairs opts: Upstairs UUID: 8b6ef580-5ae2-40fa-a8de-32e9e005b515, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:39.160403468Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1429,"session_id":"32108433-215b-4274-9965-4e63faa6859a"} {"msg":"Crucible stats registered with UUID: 8b6ef580-5ae2-40fa-a8de-32e9e005b515","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:39.160413277Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1429,"session_id":"32108433-215b-4274-9965-4e63faa6859a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:44:39.160422337Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1429,"session_id":"32108433-215b-4274-9965-4e63faa6859a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:39.160557333Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1429,"":"io task","client":"1","":"downstairs","session_id":"32108433-215b-4274-9965-4e63faa6859a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:39.160574273Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1429,"":"io task","client":"0","":"downstairs","session_id":"32108433-215b-4274-9965-4e63faa6859a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:39.160584232Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1429,"":"io task","client":"2","":"downstairs","session_id":"32108433-215b-4274-9965-4e63faa6859a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:39.160750158Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1429,"task":"crutest"} {"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:39.160766637Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1429} {"msg":"8b6ef580-5ae2-40fa-a8de-32e9e005b515 active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:39.160778487Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1429,"session_id":"32108433-215b-4274-9965-4e63faa6859a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:39.160787167Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1429,"client":"0","":"downstairs","session_id":"32108433-215b-4274-9965-4e63faa6859a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:39.160796896Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1429,"client":"1","":"downstairs","session_id":"32108433-215b-4274-9965-4e63faa6859a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:39.160806556Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1429,"client":"2","":"downstairs","session_id":"32108433-215b-4274-9965-4e63faa6859a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:39.160816086Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1429,"":"io task","client":"2","":"downstairs","session_id":"32108433-215b-4274-9965-4e63faa6859a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:39.160826045Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1429,"":"io task","client":"0","":"downstairs","session_id":"32108433-215b-4274-9965-4e63faa6859a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:39.160837655Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1429,"":"io task","client":"1","":"downstairs","session_id":"32108433-215b-4274-9965-4e63faa6859a"} {"msg":"ds_connection connected from Ok(127.0.0.1:50885)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:39.160877694Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1429,"":"io task","client":"0","":"downstairs","session_id":"32108433-215b-4274-9965-4e63faa6859a"} {"msg":"ds_connection connected from Ok(127.0.0.1:63566)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:39.160888734Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1429,"":"io task","client":"1","":"downstairs","session_id":"32108433-215b-4274-9965-4e63faa6859a"} {"msg":"ds_connection connected from Ok(127.0.0.1:55025)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:39.160948072Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1429,"":"io task","client":"2","":"downstairs","session_id":"32108433-215b-4274-9965-4e63faa6859a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:39.161616943Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1429,"client":"2","":"downstairs","session_id":"32108433-215b-4274-9965-4e63faa6859a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:39.161646722Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1429,"client":"1","":"downstairs","session_id":"32108433-215b-4274-9965-4e63faa6859a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:39.161658321Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1429,"client":"0","":"downstairs","session_id":"32108433-215b-4274-9965-4e63faa6859a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:39.161947983Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1429,"session_id":"32108433-215b-4274-9965-4e63faa6859a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:39.161957743Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1429,"session_id":"32108433-215b-4274-9965-4e63faa6859a"} {"msg":"[0]R flush_numbers[0..12]: [112, 116, 111, 110, 94, 113, 114, 110, 111, 116, 116, 116]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:39.161994152Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1429,"":"downstairs","":"downstairs","session_id":"32108433-215b-4274-9965-4e63faa6859a"} {"msg":"[0]R generation[0..12]: [72, 73, 69, 68, 57, 72, 72, 68, 69, 73, 73, 73]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:39.162005081Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1429,"":"downstairs","":"downstairs","session_id":"32108433-215b-4274-9965-4e63faa6859a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:39.162016261Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1429,"":"downstairs","":"downstairs","session_id":"32108433-215b-4274-9965-4e63faa6859a"} {"msg":"[1]R flush_numbers[0..12]: [112, 116, 111, 110, 94, 113, 114, 110, 111, 116, 116, 116]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:39.162026831Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1429,"":"downstairs","":"downstairs","session_id":"32108433-215b-4274-9965-4e63faa6859a"} {"msg":"[1]R generation[0..12]: [72, 73, 69, 68, 57, 72, 72, 68, 69, 73, 73, 73]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:39.162037101Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1429,"":"downstairs","":"downstairs","session_id":"32108433-215b-4274-9965-4e63faa6859a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:39.16205084Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1429,"":"downstairs","":"downstairs","session_id":"32108433-215b-4274-9965-4e63faa6859a"} {"msg":"[2]R flush_numbers[0..12]: [112, 116, 111, 110, 94, 113, 114, 110, 111, 116, 116, 116]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:39.16206187Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1429,"":"downstairs","":"downstairs","session_id":"32108433-215b-4274-9965-4e63faa6859a"} {"msg":"[2]R generation[0..12]: [72, 73, 69, 68, 57, 72, 72, 68, 69, 73, 73, 73]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:39.16207208Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1429,"":"downstairs","":"downstairs","session_id":"32108433-215b-4274-9965-4e63faa6859a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:39.162082869Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1429,"":"downstairs","":"downstairs","session_id":"32108433-215b-4274-9965-4e63faa6859a"} {"msg":"Max found gen is 74","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:39.162092359Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1429,"":"downstairs","":"downstairs","session_id":"32108433-215b-4274-9965-4e63faa6859a"} {"msg":"Generation requested: 74 >= found:74","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:39.162102139Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1429,"":"downstairs","":"downstairs","session_id":"32108433-215b-4274-9965-4e63faa6859a"} {"msg":"Next flush: 117","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:39.162111528Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1429,"":"downstairs","":"downstairs","session_id":"32108433-215b-4274-9965-4e63faa6859a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:39.162121008Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1429,"":"downstairs","":"downstairs","session_id":"32108433-215b-4274-9965-4e63faa6859a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:39.162130788Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1429,"session_id":"32108433-215b-4274-9965-4e63faa6859a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:39.162139508Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1429,"client":"0","":"downstairs","session_id":"32108433-215b-4274-9965-4e63faa6859a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:39.162148967Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1429,"client":"1","":"downstairs","session_id":"32108433-215b-4274-9965-4e63faa6859a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:39.162158627Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1429,"client":"2","":"downstairs","session_id":"32108433-215b-4274-9965-4e63faa6859a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:39.162168397Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1429,"session_id":"32108433-215b-4274-9965-4e63faa6859a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:39.162177466Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1429,"session_id":"32108433-215b-4274-9965-4e63faa6859a"} {"msg":"8b6ef580-5ae2-40fa-a8de-32e9e005b515 is now active with session: 32108433-215b-4274-9965-4e63faa6859a","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:39.162187386Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1429,"session_id":"32108433-215b-4274-9965-4e63faa6859a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:39.162196276Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1429,"session_id":"32108433-215b-4274-9965-4e63faa6859a"} {"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:39.162207086Z","hostname":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1429} {"msg":"Activated sub_volume 8b6ef580-5ae2-40fa-a8de-32e9e005b515","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:39.162220695Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1429} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:39.162228715Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1429,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:39.162472478Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1429,"task":"crutest"} 01/30 Write block 503 len 40960 data: 6 5 4 4 5 5 5 6 8 5 02/30 Write block 511 len 16384 data: 9 6 4 3 03/30 Write block 303 len 20480 data: 5 5 5 4 5 04/30 Read block 100 len 32768 05/30 Read block 303 len 4096 06/30 Read block 426 len 4096 07/30 Read block 538 len 12288 08/30 Write block 561 len 20480 data: 8 9 7 8 8 09/30 Read block 438 len 12288 10/30 Flush 11/30 Write block 201 len 8192 data: 10 10 12/30 Write block 204 len 24576 data: 11 9 9 7 9 9 13/30 Write block 28 len 40960 data: 3 4 5 7 7 7 7 6 5 5 14/30 Read block 176 len 12288 15/30 Read block 156 len 20480 16/30 Write block 474 len 24576 data: 3 7 9 9 9 8 17/30 Read block 166 len 40960 18/30 Write block 409 len 24576 data: 6 8 8 8 8 6 19/30 Read block 200 len 8192 20/30 Write block 547 len 36864 data: 6 8 8 9 5 7 8 5 5 21/30 Write block 78 len 40960 data: 7 6 6 5 5 6 6 5 4 3 22/30 Read block 546 len 20480 23/30 Flush 24/30 Write block 231 len 4096 data: 10 25/30 Write block 191 len 12288 data: 6 6 5 26/30 Write block 6 len 24576 data: 11 9 9 7 6 6 27/30 Read block 460 len 20480 28/30 Read block 243 len 4096 29/30 Write block 123 len 4096 data: 8 30/30 Write block 255 len 8192 data: 9 7 ---------------------------------------------------------------- Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Write 4 Sent Done Done false 1002 Acked Write 5 Sent Done Done false 1003 Acked Read 8 Sent Done Done false 1004 Acked Read 1 Sent Done Done false 1005 Acked Read 1 Sent Done Done false 1006 Acked Read 3 Sent Done Done false 1007 Acked Write 5 Sent Done Done false 1008 Acked Read 3 Sent Done Done false 1009 Acked Flush 0 Sent Done Done false 1010 Acked Write 2 Sent Done Done false 1011 Acked Write 6 Sent Done Done false 1012 Acked Write 10 Sent Done Done false 1013 Acked Read 3 Sent Done Done false 1014 Acked Read 5 Sent Done Done false 1015 Acked Write 6 Sent Done Done false 1016 Acked Read 10 Sent Done Done false 1017 Acked Write 6 Sent Done Done false 1018 Acked Read 2 Sent Done Done false 1019 Acked Write 9 Sent Done Done false 1020 Acked Write 10 Sent Done Done false 1021 Acked Read 5 Sent Done Done false 1022 Acked Flush 0 Sent Done Done false 1023 Acked Write 1 Sent Done Done false 1024 Acked Write 3 Sent Done Done false 1025 Acked Write 6 Sent Done Done false 1026 Acked Read 5 Sent Done Done false 1027 Acked Read 1 Sent Done Done false 1028 Acked Write 1 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 2 2 33 Done 1 28 28 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:44:39.170495447Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1429} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:39.170685451Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1429,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 72 72 72 112 112 112 F T T <--- 1 020-039 73 74 74 116 118 118 F F F <--- 2 040-059 69 69 69 111 111 111 F F F 3 060-079 68 74 74 110 118 118 F F F <--- 4 080-099 57 74 74 94 118 118 F F F <--- 5 100-119 72 72 72 113 113 113 F F F 6 120-139 72 72 72 114 114 114 F T T <--- 7 140-159 68 68 68 110 110 110 F F F 8 160-179 69 69 69 111 111 111 F F F 9 180-199 73 73 73 116 116 116 F T T <--- 10 200-219 73 74 74 116 118 118 F F F <--- 11 220-239 73 73 73 116 116 116 F T T <--- 12 240-259 73 73 73 116 116 116 F T T <--- 13 260-279 67 67 67 109 109 109 F F F 14 280-299 72 72 72 114 114 114 F F F 15 300-319 69 74 74 111 117 117 F F F <--- 16 320-339 72 72 72 115 115 115 F F F 17 340-359 67 67 67 109 109 109 F F F 18 360-379 69 69 69 111 111 111 F F F 19 380-399 69 69 69 111 111 111 F F F 20 400-419 72 74 74 115 118 118 F F F <--- 21 420-439 72 72 72 115 115 115 F F F 22 440-459 68 68 68 110 110 110 F F F 23 460-479 67 74 74 109 118 118 F F F <--- 24 480-499 73 73 73 116 116 116 F F F 25 500-519 68 74 74 110 117 117 T F F <--- 26 520-539 72 72 72 112 112 112 F F F 27 540-559 72 74 74 112 118 118 F F F <--- 28 560-579 69 74 74 111 117 117 F F F <--- 29 580-599 72 72 72 115 115 115 F F F Max gen: 74, Max flush: 118 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 37 Verifying data now No change after verify Loop: 37 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 75 75 75 119 119 119 F F F 1 020-039 74 74 74 118 118 118 F F F 2 040-059 69 69 69 111 111 111 F F F 3 060-079 74 74 74 118 118 118 F F F 4 080-099 74 74 74 118 118 118 F F F 5 100-119 72 72 72 113 113 113 F F F 6 120-139 75 75 75 119 119 119 F F F 7 140-159 68 68 68 110 110 110 F F F 8 160-179 69 69 69 111 111 111 F F F 9 180-199 75 75 75 119 119 119 F F F 10 200-219 74 74 74 118 118 118 F F F 11 220-239 75 75 75 119 119 119 F F F 12 240-259 75 75 75 119 119 119 F F F 13 260-279 67 67 67 109 109 109 F F F 14 280-299 72 72 72 114 114 114 F F F 15 300-319 74 74 74 117 117 117 F F F 16 320-339 72 72 72 115 115 115 F F F 17 340-359 67 67 67 109 109 109 F F F 18 360-379 69 69 69 111 111 111 F F F 19 380-399 69 69 69 111 111 111 F F F 20 400-419 74 74 74 118 118 118 F F F 21 420-439 72 72 72 115 115 115 F F F 22 440-459 68 68 68 110 110 110 F F F 23 460-479 74 74 74 118 118 118 F F F 24 480-499 73 73 73 116 116 116 F F F 25 500-519 74 74 74 117 117 117 F F F 26 520-539 72 72 72 112 112 112 F F F 27 540-559 74 74 74 118 118 118 F F F 28 560-579 74 74 74 117 117 117 F F F 29 580-599 72 72 72 115 115 115 F F F Max gen: 75, Max flush: 119 Begin loop 38 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:43.428446018Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1437,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:43.436763888Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1437,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:43.437117737Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1437} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:43.437151706Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1437} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:43.437161056Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1437} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:43.437172566Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1437} {"msg":"Crucible 09e12b9d-4916-4ef4-bb6d-b9861720b6b4 has session id: 54772162-b9b6-4fb3-9861-f4800ec8724e","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:43.437453208Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1437,"session_id":"54772162-b9b6-4fb3-9861-f4800ec8724e"} {"msg":"Upstairs opts: Upstairs UUID: 09e12b9d-4916-4ef4-bb6d-b9861720b6b4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:43.437482707Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1437,"session_id":"54772162-b9b6-4fb3-9861-f4800ec8724e"} {"msg":"Crucible stats registered with UUID: 09e12b9d-4916-4ef4-bb6d-b9861720b6b4","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:43.437493667Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1437,"session_id":"54772162-b9b6-4fb3-9861-f4800ec8724e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:44:43.437502806Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1437,"session_id":"54772162-b9b6-4fb3-9861-f4800ec8724e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:43.437638902Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1437,"":"io task","client":"1","":"downstairs","session_id":"54772162-b9b6-4fb3-9861-f4800ec8724e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:43.437657352Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1437,"":"io task","client":"0","":"downstairs","session_id":"54772162-b9b6-4fb3-9861-f4800ec8724e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:43.437667562Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1437,"":"io task","client":"2","":"downstairs","session_id":"54772162-b9b6-4fb3-9861-f4800ec8724e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:43.437804078Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1437,"task":"crutest"} {"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:43.437814747Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1437} {"msg":"09e12b9d-4916-4ef4-bb6d-b9861720b6b4 active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:43.437850416Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1437,"session_id":"54772162-b9b6-4fb3-9861-f4800ec8724e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:43.437859636Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1437,"client":"0","":"downstairs","session_id":"54772162-b9b6-4fb3-9861-f4800ec8724e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:43.437869256Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1437,"client":"1","":"downstairs","session_id":"54772162-b9b6-4fb3-9861-f4800ec8724e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:43.437878765Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1437,"client":"2","":"downstairs","session_id":"54772162-b9b6-4fb3-9861-f4800ec8724e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:43.437888385Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1437,"":"io task","client":"2","":"downstairs","session_id":"54772162-b9b6-4fb3-9861-f4800ec8724e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:43.437898385Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1437,"":"io task","client":"0","":"downstairs","session_id":"54772162-b9b6-4fb3-9861-f4800ec8724e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:43.437908145Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1437,"":"io task","client":"1","":"downstairs","session_id":"54772162-b9b6-4fb3-9861-f4800ec8724e"} {"msg":"ds_connection connected from Ok(127.0.0.1:57592)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:43.438000032Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1437,"":"io task","client":"2","":"downstairs","session_id":"54772162-b9b6-4fb3-9861-f4800ec8724e"} {"msg":"ds_connection connected from Ok(127.0.0.1:46358)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:43.438024951Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1437,"":"io task","client":"0","":"downstairs","session_id":"54772162-b9b6-4fb3-9861-f4800ec8724e"} {"msg":"ds_connection connected from Ok(127.0.0.1:45093)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:43.438129048Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1437,"":"io task","client":"1","":"downstairs","session_id":"54772162-b9b6-4fb3-9861-f4800ec8724e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:43.438687052Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1437,"client":"1","":"downstairs","session_id":"54772162-b9b6-4fb3-9861-f4800ec8724e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:43.438715981Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1437,"client":"0","":"downstairs","session_id":"54772162-b9b6-4fb3-9861-f4800ec8724e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:43.438854527Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1437,"client":"2","":"downstairs","session_id":"54772162-b9b6-4fb3-9861-f4800ec8724e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:43.439035152Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1437,"session_id":"54772162-b9b6-4fb3-9861-f4800ec8724e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:43.439045292Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1437,"session_id":"54772162-b9b6-4fb3-9861-f4800ec8724e"} {"msg":"[0]R flush_numbers[0..12]: [119, 118, 111, 118, 118, 113, 119, 110, 111, 119, 118, 119]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:43.439170928Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1437,"":"downstairs","":"downstairs","session_id":"54772162-b9b6-4fb3-9861-f4800ec8724e"} {"msg":"[0]R generation[0..12]: [75, 74, 69, 74, 74, 72, 75, 68, 69, 75, 74, 75]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:43.439182058Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1437,"":"downstairs","":"downstairs","session_id":"54772162-b9b6-4fb3-9861-f4800ec8724e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:43.439193128Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1437,"":"downstairs","":"downstairs","session_id":"54772162-b9b6-4fb3-9861-f4800ec8724e"} {"msg":"[1]R flush_numbers[0..12]: [119, 118, 111, 118, 118, 113, 119, 110, 111, 119, 118, 119]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:43.439203677Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1437,"":"downstairs","":"downstairs","session_id":"54772162-b9b6-4fb3-9861-f4800ec8724e"} {"msg":"[1]R generation[0..12]: [75, 74, 69, 74, 74, 72, 75, 68, 69, 75, 74, 75]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:43.439213917Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1437,"":"downstairs","":"downstairs","session_id":"54772162-b9b6-4fb3-9861-f4800ec8724e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:43.439224837Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1437,"":"downstairs","":"downstairs","session_id":"54772162-b9b6-4fb3-9861-f4800ec8724e"} {"msg":"[2]R flush_numbers[0..12]: [119, 118, 111, 118, 118, 113, 119, 110, 111, 119, 118, 119]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:43.439235286Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1437,"":"downstairs","":"downstairs","session_id":"54772162-b9b6-4fb3-9861-f4800ec8724e"} {"msg":"[2]R generation[0..12]: [75, 74, 69, 74, 74, 72, 75, 68, 69, 75, 74, 75]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:43.439245436Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1437,"":"downstairs","":"downstairs","session_id":"54772162-b9b6-4fb3-9861-f4800ec8724e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:43.439256196Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1437,"":"downstairs","":"downstairs","session_id":"54772162-b9b6-4fb3-9861-f4800ec8724e"} {"msg":"Max found gen is 76","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:43.439265505Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1437,"":"downstairs","":"downstairs","session_id":"54772162-b9b6-4fb3-9861-f4800ec8724e"} {"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:43.439277095Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1437,"":"downstairs","":"downstairs","session_id":"54772162-b9b6-4fb3-9861-f4800ec8724e"} {"msg":"Next flush: 120","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:43.439286365Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1437,"":"downstairs","":"downstairs","session_id":"54772162-b9b6-4fb3-9861-f4800ec8724e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:43.439295885Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1437,"":"downstairs","":"downstairs","session_id":"54772162-b9b6-4fb3-9861-f4800ec8724e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:43.439305554Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1437,"session_id":"54772162-b9b6-4fb3-9861-f4800ec8724e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:43.439314184Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1437,"client":"0","":"downstairs","session_id":"54772162-b9b6-4fb3-9861-f4800ec8724e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:43.439323794Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1437,"client":"1","":"downstairs","session_id":"54772162-b9b6-4fb3-9861-f4800ec8724e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:43.439333424Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1437,"client":"2","":"downstairs","session_id":"54772162-b9b6-4fb3-9861-f4800ec8724e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:43.439343183Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1437,"session_id":"54772162-b9b6-4fb3-9861-f4800ec8724e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:43.439352193Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1437,"session_id":"54772162-b9b6-4fb3-9861-f4800ec8724e"} {"msg":"09e12b9d-4916-4ef4-bb6d-b9861720b6b4 is now active with session: 54772162-b9b6-4fb3-9861-f4800ec8724e","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:43.439362033Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1437,"session_id":"54772162-b9b6-4fb3-9861-f4800ec8724e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:43.439370782Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1437,"session_id":"54772162-b9b6-4fb3-9861-f4800ec8724e"} {"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:43.439379802Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1437} {"msg":"Activated sub_volume 09e12b9d-4916-4ef4-bb6d-b9861720b6b4","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:43.439387752Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1437} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:43.439395782Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1437,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:43.439705493Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1437,"task":"crutest"} 01/30 Write block 444 len 36864 data: 3 3 3 3 3 3 5 5 4 02/30 Read block 275 len 4096 03/30 Write block 255 len 36864 data: 10 8 5 5 5 5 5 5 4 04/30 Write block 289 len 32768 data: 6 6 6 7 8 9 8 8 05/30 Write block 58 len 20480 data: 8 6 5 7 7 06/30 Read block 110 len 24576 07/30 Read block 491 len 36864 08/30 Write block 61 len 12288 data: 8 8 6 09/30 Write block 514 len 36864 data: 4 4 4 4 6 8 8 9 9 10/30 Write block 397 len 40960 data: 6 5 6 6 6 6 6 7 6 5 11/30 Write block 517 len 36864 data: 5 7 9 9 10 10 10 9 8 12/30 Write block 345 len 20480 data: 7 6 6 5 5 13/30 Flush 14/30 Write block 115 len 12288 data: 6 7 7 15/30 Write block 442 len 20480 data: 5 4 4 4 4 16/30 Flush 17/30 Write block 498 len 36864 data: 5 5 6 6 6 7 6 5 5 18/30 Read block 268 len 24576 19/30 Flush 20/30 Write block 71 len 40960 data: 4 4 4 4 5 5 6 8 7 7 21/30 Read block 396 len 8192 22/30 Flush 23/30 Write block 386 len 32768 data: 7 7 6 5 5 5 5 5 24/30 Write block 510 len 4096 data: 7 25/30 Flush 26/30 Write block 571 len 16384 data: 7 5 6 6 27/30 Read block 104 len 4096 28/30 Write block 467 len 32768 data: 3 5 6 6 4 3 2 4 29/30 Write block 587 len 16384 data: 8 8 5 5 30/30 Read block 8 len 24576 ---------------------------------------------------------------- Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Read 1 Done Done Done false 1002 Acked Write 9 Done Done Done false 1003 Acked Write 8 Done Done Done false 1004 Acked Write 5 Done Done Done false 1005 Acked Read 6 Done Done Done false 1006 Acked Read 9 Done Done Done false 1007 Acked Write 3 Done Done Done false 1008 Acked Write 9 Done Done Sent false 1009 Acked Write 10 Done Done Sent false 1010 Acked Write 9 Done Done Sent false 1011 Acked Write 5 Done Done Sent false 1012 Acked Flush 0 Done Done Sent false 1013 Acked Write 3 Done Done Sent false 1014 Acked Write 5 Done Done Sent false 1015 Acked Flush 0 Done Done Sent false 1016 Acked Write 9 Done Done Sent false 1017 Acked Read 6 Done Done Sent false 1018 Acked Flush 0 Done Done Sent false 1019 Acked Write 10 Done Done Sent false 1020 Acked Read 2 Done Done Sent false 1021 Acked Flush 0 Done Done Sent false 1022 Acked Write 8 Done Done Sent false 1023 Acked Write 1 Done Done Sent false 1024 Acked Flush 0 Done Done Sent false 1025 Acked Write 4 Done Done Sent false 1026 Acked Read 1 Done Done Sent false 1027 Acked Write 8 Done Done Sent false 1028 Acked Write 4 Done Done Sent false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 22 23 Done 30 29 8 67 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:44:43.448716833Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1437} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:43.448887668Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1437,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 75 75 75 119 119 119 F F F 1 020-039 74 74 74 118 118 118 F F F 2 040-059 76 76 69 120 120 111 F F T <--- 3 060-079 76 76 74 123 123 118 F F T <--- 4 080-099 76 76 74 123 123 118 F F F <--- 5 100-119 76 76 72 121 121 113 F F F <--- 6 120-139 75 75 75 119 119 119 F F F 7 140-159 68 68 68 110 110 110 F F F 8 160-179 69 69 69 111 111 111 F F F 9 180-199 75 75 75 119 119 119 F F F 10 200-219 74 74 74 118 118 118 F F F 11 220-239 75 75 75 119 119 119 F F F 12 240-259 76 76 75 120 120 119 F F T <--- 13 260-279 76 76 67 120 120 109 F F T <--- 14 280-299 76 76 72 120 120 114 F F T <--- 15 300-319 74 74 74 117 117 117 F F F 16 320-339 72 72 72 115 115 115 F F F 17 340-359 76 76 67 120 120 109 F F F <--- 18 360-379 69 69 69 111 111 111 F F F 19 380-399 76 76 69 124 124 111 F F F <--- 20 400-419 76 76 74 120 120 118 F F F <--- 21 420-439 72 72 72 115 115 115 F F F 22 440-459 76 76 68 121 121 110 F F T <--- 23 460-479 74 74 74 118 118 118 T T F <--- 24 480-499 76 76 73 122 122 116 F F F <--- 25 500-519 76 76 74 124 124 117 F F F <--- 26 520-539 76 76 72 120 120 112 F F F <--- 27 540-559 74 74 74 118 118 118 F F F 28 560-579 74 74 74 117 117 117 T T F <--- 29 580-599 72 72 72 115 115 115 T T F <--- Max gen: 76, Max flush: 124 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 38 Verifying data now No change after verify Loop: 38 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 75 75 75 119 119 119 F F F 1 020-039 74 74 74 118 118 118 F F F 2 040-059 76 76 76 120 120 120 F F F 3 060-079 76 76 76 123 123 123 F F F 4 080-099 76 76 76 123 123 123 F F F 5 100-119 76 76 76 121 121 121 F F F 6 120-139 75 75 75 119 119 119 F F F 7 140-159 68 68 68 110 110 110 F F F 8 160-179 69 69 69 111 111 111 F F F 9 180-199 75 75 75 119 119 119 F F F 10 200-219 74 74 74 118 118 118 F F F 11 220-239 75 75 75 119 119 119 F F F 12 240-259 76 76 76 120 120 120 F F F 13 260-279 76 76 76 120 120 120 F F F 14 280-299 76 76 76 120 120 120 F F F 15 300-319 74 74 74 117 117 117 F F F 16 320-339 72 72 72 115 115 115 F F F 17 340-359 76 76 76 120 120 120 F F F 18 360-379 69 69 69 111 111 111 F F F 19 380-399 76 76 76 124 124 124 F F F 20 400-419 76 76 76 120 120 120 F F F 21 420-439 72 72 72 115 115 115 F F F 22 440-459 76 76 76 121 121 121 F F F 23 460-479 77 77 77 125 125 125 F F F 24 480-499 76 76 76 122 122 122 F F F 25 500-519 76 76 76 124 124 124 F F F 26 520-539 76 76 76 120 120 120 F F F 27 540-559 74 74 74 118 118 118 F F F 28 560-579 77 77 77 125 125 125 F F F 29 580-599 77 77 77 125 125 125 F F F Max gen: 77, Max flush: 125 Begin loop 39 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:47.703902794Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1445,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:47.71201273Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1445,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:47.712303241Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1445} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:47.712328251Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1445} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:47.712337Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1445} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:47.71234871Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1445} {"msg":"Crucible cc274e03-d7e0-4d18-b012-b6d9d80f963b has session id: a2ac295d-03e3-4623-af2c-27c86fef507b","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:47.712658621Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1445,"session_id":"a2ac295d-03e3-4623-af2c-27c86fef507b"} {"msg":"Upstairs opts: Upstairs UUID: cc274e03-d7e0-4d18-b012-b6d9d80f963b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:47.712720129Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1445,"session_id":"a2ac295d-03e3-4623-af2c-27c86fef507b"} {"msg":"Crucible stats registered with UUID: cc274e03-d7e0-4d18-b012-b6d9d80f963b","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:47.712730559Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1445,"session_id":"a2ac295d-03e3-4623-af2c-27c86fef507b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:44:47.712739619Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1445,"session_id":"a2ac295d-03e3-4623-af2c-27c86fef507b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:47.712877505Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1445,"":"io task","client":"1","":"downstairs","session_id":"a2ac295d-03e3-4623-af2c-27c86fef507b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:47.712911564Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1445,"":"io task","client":"0","":"downstairs","session_id":"a2ac295d-03e3-4623-af2c-27c86fef507b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:47.712922423Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1445,"":"io task","client":"2","":"downstairs","session_id":"a2ac295d-03e3-4623-af2c-27c86fef507b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:47.71304442Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1445,"task":"crutest"} {"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:47.713060399Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1445} {"msg":"cc274e03-d7e0-4d18-b012-b6d9d80f963b active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:47.713098888Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1445,"session_id":"a2ac295d-03e3-4623-af2c-27c86fef507b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:47.713108338Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1445,"client":"0","":"downstairs","session_id":"a2ac295d-03e3-4623-af2c-27c86fef507b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:47.713117948Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1445,"client":"1","":"downstairs","session_id":"a2ac295d-03e3-4623-af2c-27c86fef507b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:47.713127428Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1445,"client":"2","":"downstairs","session_id":"a2ac295d-03e3-4623-af2c-27c86fef507b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:47.713136847Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1445,"":"io task","client":"2","":"downstairs","session_id":"a2ac295d-03e3-4623-af2c-27c86fef507b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:47.713146717Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1445,"":"io task","client":"0","":"downstairs","session_id":"a2ac295d-03e3-4623-af2c-27c86fef507b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:47.713161177Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1445,"":"io task","client":"1","":"downstairs","session_id":"a2ac295d-03e3-4623-af2c-27c86fef507b"} {"msg":"ds_connection connected from Ok(127.0.0.1:37508)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:47.713227375Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1445,"":"io task","client":"2","":"downstairs","session_id":"a2ac295d-03e3-4623-af2c-27c86fef507b"} {"msg":"ds_connection connected from Ok(127.0.0.1:46160)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:47.713241074Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1445,"":"io task","client":"0","":"downstairs","session_id":"a2ac295d-03e3-4623-af2c-27c86fef507b"} {"msg":"ds_connection connected from Ok(127.0.0.1:44232)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:47.713251454Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1445,"":"io task","client":"1","":"downstairs","session_id":"a2ac295d-03e3-4623-af2c-27c86fef507b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:47.713959164Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1445,"client":"0","":"downstairs","session_id":"a2ac295d-03e3-4623-af2c-27c86fef507b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:47.713977373Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1445,"client":"1","":"downstairs","session_id":"a2ac295d-03e3-4623-af2c-27c86fef507b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:47.714137828Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1445,"client":"2","":"downstairs","session_id":"a2ac295d-03e3-4623-af2c-27c86fef507b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:47.714265345Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1445,"session_id":"a2ac295d-03e3-4623-af2c-27c86fef507b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:47.714283694Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1445,"session_id":"a2ac295d-03e3-4623-af2c-27c86fef507b"} {"msg":"[0]R flush_numbers[0..12]: [119, 118, 120, 123, 123, 121, 119, 110, 111, 119, 118, 119]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:47.71443976Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1445,"":"downstairs","":"downstairs","session_id":"a2ac295d-03e3-4623-af2c-27c86fef507b"} {"msg":"[0]R generation[0..12]: [75, 74, 76, 76, 76, 76, 75, 68, 69, 75, 74, 75]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:47.714450899Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1445,"":"downstairs","":"downstairs","session_id":"a2ac295d-03e3-4623-af2c-27c86fef507b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:47.714461799Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1445,"":"downstairs","":"downstairs","session_id":"a2ac295d-03e3-4623-af2c-27c86fef507b"} {"msg":"[1]R flush_numbers[0..12]: [119, 118, 120, 123, 123, 121, 119, 110, 111, 119, 118, 119]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:47.714472399Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1445,"":"downstairs","":"downstairs","session_id":"a2ac295d-03e3-4623-af2c-27c86fef507b"} {"msg":"[1]R generation[0..12]: [75, 74, 76, 76, 76, 76, 75, 68, 69, 75, 74, 75]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:47.714482798Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1445,"":"downstairs","":"downstairs","session_id":"a2ac295d-03e3-4623-af2c-27c86fef507b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:47.714493638Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1445,"":"downstairs","":"downstairs","session_id":"a2ac295d-03e3-4623-af2c-27c86fef507b"} {"msg":"[2]R flush_numbers[0..12]: [119, 118, 120, 123, 123, 121, 119, 110, 111, 119, 118, 119]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:47.714504288Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1445,"":"downstairs","":"downstairs","session_id":"a2ac295d-03e3-4623-af2c-27c86fef507b"} {"msg":"[2]R generation[0..12]: [75, 74, 76, 76, 76, 76, 75, 68, 69, 75, 74, 75]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:47.714514468Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1445,"":"downstairs","":"downstairs","session_id":"a2ac295d-03e3-4623-af2c-27c86fef507b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:47.714525187Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1445,"":"downstairs","":"downstairs","session_id":"a2ac295d-03e3-4623-af2c-27c86fef507b"} {"msg":"Max found gen is 78","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:47.714534617Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1445,"":"downstairs","":"downstairs","session_id":"a2ac295d-03e3-4623-af2c-27c86fef507b"} {"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:47.714544177Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1445,"":"downstairs","":"downstairs","session_id":"a2ac295d-03e3-4623-af2c-27c86fef507b"} {"msg":"Next flush: 126","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:47.714553536Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1445,"":"downstairs","":"downstairs","session_id":"a2ac295d-03e3-4623-af2c-27c86fef507b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:47.714562856Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1445,"":"downstairs","":"downstairs","session_id":"a2ac295d-03e3-4623-af2c-27c86fef507b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:47.714572596Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1445,"session_id":"a2ac295d-03e3-4623-af2c-27c86fef507b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:47.714581326Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1445,"client":"0","":"downstairs","session_id":"a2ac295d-03e3-4623-af2c-27c86fef507b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:47.714592195Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1445,"client":"1","":"downstairs","session_id":"a2ac295d-03e3-4623-af2c-27c86fef507b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:47.714601815Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1445,"client":"2","":"downstairs","session_id":"a2ac295d-03e3-4623-af2c-27c86fef507b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:47.714611585Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1445,"session_id":"a2ac295d-03e3-4623-af2c-27c86fef507b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:47.714620654Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1445,"session_id":"a2ac295d-03e3-4623-af2c-27c86fef507b"} {"msg":"cc274e03-d7e0-4d18-b012-b6d9d80f963b is now active with session: a2ac295d-03e3-4623-af2c-27c86fef507b","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:47.714630584Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1445,"session_id":"a2ac295d-03e3-4623-af2c-27c86fef507b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:47.714639294Z",Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1445,"session_id":"a2ac295d-03e3-4623-af2c-27c86fef507b"} {"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:47.714653513Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1445} {"msg":"Activated sub_volume cc274e03-d7e0-4d18-b012-b6d9d80f963b","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:47.714661773Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1445} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:47.714669983Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1445,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:47.714906176Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1445,"task":"crutest"} 01/30 Write block 424 len 24576 data: 5 4 3 4 3 4 02/30 Read block 159 len 16384 03/30 Write block 374 len 40960 data: 2 6 10 10 11 12 11 11 11 8 04/30 Read block 175 len 28672 05/30 Read block 152 len 28672 06/30 Write block 504 len 16384 data: 7 6 6 6 07/30 Read block 470 len 36864 08/30 Read block 77 len 12288 09/30 Write block 487 len 40960 data: 8 8 6 7 8 6 5 4 6 7 10/30 Write block 362 len 24576 data: 9 10 11 11 9 9 11/30 Write block 233 len 16384 data: 8 8 5 4 12/30 Flush 13/30 Write block 583 len 20480 data: 7 7 7 7 9 14/30 Read block 245 len 8192 15/30 Read block 135 len 8192 16/30 Write block 177 len 40960 data: 7 6 5 4 3 4 4 7 6 7 17/30 Read block 567 len 8192 18/30 Read block 107 len 40960 19/30 Read block 311 len 12288 20/30 Read block 212 len 8192 21/30 Write block 31 len 12288 data: 8 8 8 22/30 Write block 489 len 16384 data: 7 8 9 7 23/30 Write block 70 len 40960 data: 4 5 5 5 5 6 6 7 9 8 24/30 Write block 74 len 12288 data: 6 7 7 25/30 Read block 401 len 12288 26/30 Write block 179 len 36864 data: 6 5 4 5 5 8 7 8 8 27/30 Read block 496 len 24576 28/30 Read block 225 len 24576 29/30 Read block 259 len 20480 30/30 Write block 341 len 16384 data: 6 5 6 7 ---------------------------------------------------------------- Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Sent Done false 1001 Acked Read 4 Done Sent Done false 1002 Acked Write 10 Done Sent Done false 1003 Acked Read 7 Done Sent Done false 1004 Acked Read 7 Done Sent Done false 1005 Acked Write 4 Done Sent Done false 1006 Acked Read 9 Done Sent Done false 1007 Acked Read 3 Done Sent Done false 1008 Acked Write 10 Done Sent Done false 1009 Acked Write 6 Done Sent Done false 1010 Acked Write 4 Done Sent Done false 1011 Acked Flush 0 Done Sent Done false 1012 Acked Write 5 Done Sent Done false 1013 Acked Read 2 Done Sent Done false 1014 Acked Read 2 Done Sent Done false 1015 Acked Write 10 Done Sent Done false 1016 Acked Read 2 Done Sent Done false 1017 Acked Read 10 Done Sent Done false 1018 Acked Read 3 Done Sent Done false 1019 Acked Read 2 Done Sent Done false 1020 Acked Write 3 Done Sent Done false 1021 Acked Write 4 Done Sent Done false 1022 Acked Write 10 Done Sent Done false 1023 Acked Write 3 Done Sent Done false 1024 Acked Read 3 Done Sent Done false 1025 Acked Write 9 Done Sent Done false 1026 Acked Read 6 Done Sent Done false 1027 Acked Read 6 Done Sent Done false 1028 Acked Read 5 Done Sent Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 30 1 32 Done 29 0 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1011)) None Some(JobId(1011)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:44:47.723838389Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1445} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:47.724016733Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1445,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 75 75 75 119 119 119 F F F 1 020-039 74 74 74 118 118 118 T F T <--- 2 040-059 76 76 76 120 120 120 F F F 3 060-079 76 76 76 123 123 123 T F T <--- 4 080-099 76 76 76 123 123 123 F F F 5 100-119 76 76 76 121 121 121 F F F 6 120-139 75 75 75 119 119 119 F F F 7 140-159 68 68 68 110 110 110 F F F 8 160-179 69 69 69 111 111 111 T F T <--- 9 180-199 75 75 75 119 119 119 T F T <--- 10 200-219 74 74 74 118 118 118 F F F 11 220-239 78 75 78 126 119 126 F F F <--- 12 240-259 76 76 76 120 120 120 F F F 13 260-279 76 76 76 120 120 120 F F F 14 280-299 76 76 76 120 120 120 F F F 15 300-319 74 74 74 117 117 117 F F F 16 320-339 72 72 72 115 115 115 F F F 17 340-359 76 76 76 120 120 120 T F T <--- 18 360-379 78 69 78 126 111 126 F F F <--- 19 380-399 78 76 78 126 124 126 F F F <--- 20 400-419 76 76 76 120 120 120 F F F 21 420-439 78 72 78 126 115 126 F F F <--- 22 440-459 76 76 76 121 121 121 F F F 23 460-479 77 77 77 125 125 125 F F F 24 480-499 78 76 78 126 122 126 T F T <--- 25 500-519 78 76 78 126 124 126 F F F <--- 26 520-539 76 76 76 120 120 120 F F F 27 540-559 74 74 74 118 118 118 F F F 28 560-579 77 77 77 125 125 125 F F F 29 580-599 77 77 77 125 125 125 T F T <--- Max gen: 78, Max flush: 126 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 39 Verifying data now No change after verify Loop: 39 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 75 75 75 119 119 119 F F F 1 020-039 79 79 79 127 127 127 F F F 2 040-059 76 76 76 120 120 120 F F F 3 060-079 79 79 79 127 127 127 F F F 4 080-099 76 76 76 123 123 123 F F F 5 100-119 76 76 76 121 121 121 F F F 6 120-139 75 75 75 119 119 119 F F F 7 140-159 68 68 68 110 110 110 F F F 8 160-179 79 79 79 127 127 127 F F F 9 180-199 79 79 79 127 127 127 F F F 10 200-219 74 74 74 118 118 118 F F F 11 220-239 78 78 78 126 126 126 F F F 12 240-259 76 76 76 120 120 120 F F F 13 260-279 76 76 76 120 120 120 F F F 14 280-299 76 76 76 120 120 120 F F F 15 300-319 74 74 74 117 117 117 F F F 16 320-339 72 72 72 115 115 115 F F F 17 340-359 79 79 79 127 127 127 F F F 18 360-379 78 78 78 126 126 126 F F F 19 380-399 78 78 78 126 126 126 F F F 20 400-419 76 76 76 120 120 120 F F F 21 420-439 78 78 78 126 126 126 F F F 22 440-459 76 76 76 121 121 121 F F F 23 460-479 77 77 77 125 125 125 F F F 24 480-499 79 79 79 127 127 127 F F F 25 500-519 78 78 78 126 126 126 F F F 26 520-539 76 76 76 120 120 120 F F F 27 540-559 74 74 74 118 118 118 F F F 28 560-579 77 77 77 125 125 125 F F F 29 580-599 79 79 79 127 127 127 F F F Max gen: 79, Max flush: 127 Begin loop 40 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:51.966906648Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1453,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:51.974965006Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1453,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:51.975278287Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1453} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:51.975305616Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1453} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:51.975313896Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1453} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:51.975327665Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1453} {"msg":"Crucible 491ae440-1a4e-43ae-aa02-1e055d58cd5e has session id: 8cbb8067-c48f-4cf4-ad4c-f29522cd4e72","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:51.975653036Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1453,"session_id":"8cbb8067-c48f-4cf4-ad4c-f29522cd4e72"} {"msg":"Upstairs opts: Upstairs UUID: 491ae440-1a4e-43ae-aa02-1e055d58cd5e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:51.975691845Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1453,"session_id":"8cbb8067-c48f-4cf4-ad4c-f29522cd4e72"} {"msg":"Crucible stats registered with UUID: 491ae440-1a4e-43ae-aa02-1e055d58cd5e","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:51.975701865Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1453,"session_id":"8cbb8067-c48f-4cf4-ad4c-f29522cd4e72"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:44:51.975742943Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1453,"session_id":"8cbb8067-c48f-4cf4-ad4c-f29522cd4e72"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:51.975887149Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1453,"":"io task","client":"0","":"downstairs","session_id":"8cbb8067-c48f-4cf4-ad4c-f29522cd4e72"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:51.975904309Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1453,"":"io task","client":"2","":"downstairs","session_id":"8cbb8067-c48f-4cf4-ad4c-f29522cd4e72"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:51.975914229Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1453,"":"io task","client":"1","":"downstairs","session_id":"8cbb8067-c48f-4cf4-ad4c-f29522cd4e72"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:51.976092283Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1453,"task":"crutest"} {"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:51.976107893Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1453} {"msg":"491ae440-1a4e-43ae-aa02-1e055d58cd5e active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:51.976123623Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1453,"session_id":"8cbb8067-c48f-4cf4-ad4c-f29522cd4e72"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:51.976132392Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1453,"client":"0","":"downstairs","session_id":"8cbb8067-c48f-4cf4-ad4c-f29522cd4e72"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:51.976142032Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1453,"client":"1","":"downstairs","session_id":"8cbb8067-c48f-4cf4-ad4c-f29522cd4e72"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:51.976151492Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1453,"client":"2","":"downstairs","session_id":"8cbb8067-c48f-4cf4-ad4c-f29522cd4e72"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:51.976160931Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1453,"":"io task","client":"2","":"downstairs","session_id":"8cbb8067-c48f-4cf4-ad4c-f29522cd4e72"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:51.976172281Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1453,"":"io task","client":"0","":"downstairs","session_id":"8cbb8067-c48f-4cf4-ad4c-f29522cd4e72"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:51.976182201Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1453,"":"io task","client":"1","":"downstairs","session_id":"8cbb8067-c48f-4cf4-ad4c-f29522cd4e72"} {"msg":"ds_connection connected from Ok(127.0.0.1:38940)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:51.976231999Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1453,"":"io task","client":"0","":"downstairs","session_id":"8cbb8067-c48f-4cf4-ad4c-f29522cd4e72"} {"msg":"ds_connection connected from Ok(127.0.0.1:61971)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:51.976242889Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1453,"":"io task","client":"1","":"downstairs","session_id":"8cbb8067-c48f-4cf4-ad4c-f29522cd4e72"} {"msg":"ds_connection connected from Ok(127.0.0.1:57702)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:51.976253399Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1453,"":"io task","client":"2","":"downstairs","session_id":"8cbb8067-c48f-4cf4-ad4c-f29522cd4e72"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:51.976933369Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1453,"client":"1","":"downstairs","session_id":"8cbb8067-c48f-4cf4-ad4c-f29522cd4e72"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:51.976946109Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1453,"client":"0","":"downstairs","session_id":"8cbb8067-c48f-4cf4-ad4c-f29522cd4e72"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:51.976956818Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1453,"client":"2","":"downstairs","session_id":"8cbb8067-c48f-4cf4-ad4c-f29522cd4e72"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:51.977288749Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1453,"session_id":"8cbb8067-c48f-4cf4-ad4c-f29522cd4e72"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:51.977300039Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1453,"session_id":"8cbb8067-c48f-4cf4-ad4c-f29522cd4e72"} {"msg":"[0]R flush_numbers[0..12]: [119, 127, 120, 127, 123, 121, 119, 110, 127, 127, 118, 126]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:51.977336388Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1453,"":"downstairs","":"downstairs","session_id":"8cbb8067-c48f-4cf4-ad4c-f29522cd4e72"} {"msg":"[0]R generation[0..12]: [75, 79, 76, 79, 76, 76, 75, 68, 79, 79, 74, 78]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:51.977347377Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1453,"":"downstairs","":"downstairs","session_id":"8cbb8067-c48f-4cf4-ad4c-f29522cd4e72"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:51.977358167Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1453,"":"downstairs","":"downstairs","session_id":"8cbb8067-c48f-4cf4-ad4c-f29522cd4e72"} {"msg":"[1]R flush_numbers[0..12]: [119, 127, 120, 127, 123, 121, 119, 110, 127, 127, 118, 126]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:51.977368747Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1453,"":"downstairs","":"downstairs","session_id":"8cbb8067-c48f-4cf4-ad4c-f29522cd4e72"} {"msg":"[1]R generation[0..12]: [75, 79, 76, 79, 76, 76, 75, 68, 79, 79, 74, 78]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:51.977379056Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1453,"":"downstairs","":"downstairs","session_id":"8cbb8067-c48f-4cf4-ad4c-f29522cd4e72"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:51.977391806Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1453,"":"downstairs","":"downstairs","session_id":"8cbb8067-c48f-4cf4-ad4c-f29522cd4e72"} {"msg":"[2]R flush_numbers[0..12]: [119, 127, 120, 127, 123, 121, 119, 110, 127, 127, 118, 126]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:51.977402256Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1453,"":"downstairs","":"downstairs","session_id":"8cbb8067-c48f-4cf4-ad4c-f29522cd4e72"} {"msg":"[2]R generation[0..12]: [75, 79, 76, 79, 76, 76, 75, 68, 79, 79, 74, 78]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:51.977412485Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1453,"":"downstairs","":"downstairs","session_id":"8cbb8067-c48f-4cf4-ad4c-f29522cd4e72"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:51.977423205Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1453,"":"downstairs","":"downstairs","session_id":"8cbb8067-c48f-4cf4-ad4c-f29522cd4e72"} {"msg":"Max found gen is 80","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:51.977432695Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1453,"":"downstairs","":"downstairs","session_id":"8cbb8067-c48f-4cf4-ad4c-f29522cd4e72"} {"msg":"Generation requested: 80 >= found:80","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:51.977442274Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1453,"":"downstairs","":"downstairs","session_id":"8cbb8067-c48f-4cf4-ad4c-f29522cd4e72"} {"msg":"Next flush: 128","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:51.977451584Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1453,"":"downstairs","":"downstairs","session_id":"8cbb8067-c48f-4cf4-ad4c-f29522cd4e72"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:51.977460964Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1453,"":"downstairs","":"downstairs","session_id":"8cbb8067-c48f-4cf4-ad4c-f29522cd4e72"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:51.977470574Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1453,"session_id":"8cbb8067-c48f-4cf4-ad4c-f29522cd4e72"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:51.977479233Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1453,"client":"0","":"downstairs","session_id":"8cbb8067-c48f-4cf4-ad4c-f29522cd4e72"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:51.977488983Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1453,"client":"1","":"downstairs","session_id":"8cbb8067-c48f-4cf4-ad4c-f29522cd4e72"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:51.977498593Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1453,"client":"2","":"downstairs","session_id":"8cbb8067-c48f-4cf4-ad4c-f29522cd4e72"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:51.977508343Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1453,"session_id":"8cbb8067-c48f-4cf4-ad4c-f29522cd4e72"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:51.977517492Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1453,"session_id":"8cbb8067-c48f-4cf4-ad4c-f29522cd4e72"} {"msg":"491ae440-1a4e-43ae-aa02-1e055d58cd5e is now active with session: 8cbb8067-c48f-4cf4-ad4c-f29522cd4e72","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:51.977527422Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1453,"session_id":"8cbb8067-c48f-4cf4-ad4c-f29522cd4e72"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:51.977537522Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1453,"session_id":"8cbb8067-c48f-4cf4-ad4c-f29522cd4e72"} {"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:51.977546501Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1453} {"msg":"Activated sub_volume 491ae440-1a4e-43ae-aa02-1e055d58cd5e","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:51.977554831Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1453} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:51.977562981Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1453,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:51.977896561Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1453,"task":"crutest"} 01/30 Write block 240 len 32768 data: 6 6 5 4 5 5 7 8 02/30 Read block 301 len 16384 03/30 Flush 04/30 Write block 247 len 4096 data: 9 05/30 Read block 400 len 40960 06/30 Read block 74 len 32768 07/30 Write block 533 len 12288 data: 8 8 12 08/30 Write block 530 len 20480 data: 5 5 10 9 9 09/30 Write block 353 len 4096 data: 5 10/30 Read block 497 len 8192 11/30 Read block 431 len 4096 12/30 Write block 537 len 12288 data: 10 11 11 13/30 Write block 521 len 8192 data: 11 11 14/30 Write block 571 len 4096 data: 8 15/30 Write block 185 len 32768 data: 8 9 9 7 7 6 7 7 16/30 Read block 356 len 8192 17/30 Read block 397 len 32768 18/30 Write block 290 len 32768 data: 7 7 8 9 10 9 9 10 19/30 Write block 569 len 4096 data: 6 20/30 Write block 311 len 16384 data: 6 7 7 6 21/30 Flush 22/30 Write block 507 len 20480 data: 7 6 6 8 10 23/30 Flush 24/30 Flush 25/30 Write block 73 len 28672 data: 6 7 8 8 8 10 9 26/30 Read block 209 len 16384 27/30 Write block 537 len 40960 data: 11 12 12 11 8 6 6 7 7 6 28/30 Write block 127 len 16384 data: 7 7 5 6 29/30 Read block 193 len 32768 30/30 Read block 14 len 12288 ---------------------------------------------------------------- Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1003 Acked Write 1 Done Done Sent false 1004 Acked Read 10 Done Done Sent false 1005 Acked Read 8 Done Done Sent false 1006 Acked Write 3 Done Done Sent false 1007 Acked Write 5 Done Done Sent false 1008 Acked Write 1 Done Done Sent false 1009 Acked Read 2 Done Done Sent false 1010 Acked Read 1 Done Done Sent false 1011 Acked Write 3 Done Done Sent false 1012 Acked Write 2 Done Done Sent false 1013 Acked Write 1 Done Done Sent false 1014 Acked Write 8 Done Done Sent false 1015 Acked Read 2 Done Done Sent false 1016 Acked Read 8 Done Done Sent false 1017 Acked Write 8 Done Done Sent false 1018 Acked Write 1 Done Done Sent false 1019 Acked Write 4 Done Done Sent false 1020 Acked Flush 0 Done Done Sent false 1021 Acked Write 5 Done Done Sent false 1022 Acked Flush 0 Done Done Sent false 1023 Acked Flush 0 Done Done Sent false 1024 Acked Write 7 Done Done Sent false 1025 Acked Read 4 Done Done Sent false 1026 Acked Write 10 Done Done Sent false 1027 Acked Write 4 Done Done Sent false 1028 Acked Read 8 Done Done Sent false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 27 28 Done 27 26 0 53 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1002)) Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:44:51.986590891Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1453} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:51.986772435Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1453,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 75 75 75 119 119 119 F F F 1 020-039 79 79 79 127 127 127 F F F 2 040-059 76 76 76 120 120 120 F F F 3 060-079 79 79 79 127 127 127 T T F <--- 4 080-099 76 76 76 123 123 123 F F F 5 100-119 76 76 76 121 121 121 F F F 6 120-139 75 75 75 119 119 119 T T F <--- 7 140-159 68 68 68 110 110 110 F F F 8 160-179 79 79 79 127 127 127 F F F 9 180-199 80 80 79 129 129 127 F F F <--- 10 200-219 74 74 74 118 118 118 F F F 11 220-239 78 78 78 126 126 126 F F F 12 240-259 80 80 80 129 129 128 F F F <--- 13 260-279 76 76 76 120 120 120 F F F 14 280-299 80 80 76 129 129 120 F F F <--- 15 300-319 80 80 74 129 129 117 F F F <--- 16 320-339 72 72 72 115 115 115 F F F 17 340-359 80 80 79 129 129 127 F F F <--- 18 360-379 78 78 78 126 126 126 F F F 19 380-399 78 78 78 126 126 126 F F F 20 400-419 76 76 76 120 120 120 F F F 21 420-439 78 78 78 126 126 126 F F F 22 440-459 76 76 76 121 121 121 F F F 23 460-479 77 77 77 125 125 125 F F F 24 480-499 79 79 79 127 127 127 F F F 25 500-519 80 80 78 130 130 126 F F F <--- 26 520-539 80 80 76 129 129 120 T T F <--- 27 540-559 74 74 74 118 118 118 T T F <--- 28 560-579 80 80 77 129 129 125 F F F <--- 29 580-599 79 79 79 127 127 127 F F F Max gen: 80, Max flush: 130 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 40 Verifying data now No change after verify Loop: 40 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 75 75 75 119 119 119 F F F 1 020-039 79 79 79 127 127 127 F F F 2 040-059 76 76 76 120 120 120 F F F 3 060-079 81 81 81 131 131 131 F F F 4 080-099 76 76 76 123 123 123 F F F 5 100-119 76 76 76 121 121 121 F F F 6 120-139 81 81 81 131 131 131 F F F 7 140-159 68 68 68 110 110 110 F F F 8 160-179 79 79 79 127 127 127 F F F 9 180-199 80 80 80 129 129 129 F F F 10 200-219 74 74 74 118 118 118 F F F 11 220-239 78 78 78 126 126 126 F F F 12 240-259 80 80 80 129 129 129 F F F 13 260-279 76 76 76 120 120 120 F F F 14 280-299 80 80 80 129 129 129 F F F 15 300-319 80 80 80 129 129 129 F F F 16 320-339 72 72 72 115 115 115 F F F 17 340-359 80 80 80 129 129 129 F F F 18 360-379 78 78 78 126 126 126 F F F 19 380-399 78 78 78 126 126 126 F F F 20 400-419 76 76 76 120 120 120 F F F 21 420-439 78 78 78 126 126 126 F F F 22 440-459 76 76 76 121 121 121 F F F 23 460-479 77 77 77 125 125 125 F F F 24 480-499 79 79 79 127 127 127 F F F 25 500-519 80 80 80 130 130 130 F F F 26 520-539 81 81 81 131 131 131 F F F 27 540-559 81 81 81 131 131 131 F F F 28 560-579 80 80 80 129 129 129 F F F 29 580-599 79 79 79 127 127 127 F F F Max gen: 81, Max flush: 131 Begin loop 41 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:56.230073659Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1461,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:56.238140706Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1461,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:56.238442268Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1461} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:56.238467717Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1461} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:56.238476297Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1461} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:56.238487806Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1461} {"msg":"Crucible 916f8fcf-8aab-4439-b871-4730b03ade1a has session id: e6333dc0-ca22-47ae-bf2f-7d015b5a7fb3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:56.238833966Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1461,"session_id":"e6333dc0-ca22-47ae-bf2f-7d015b5a7fb3"} {"msg":"Upstairs opts: Upstairs UUID: 916f8fcf-8aab-4439-b871-4730b03ade1a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:56.238852186Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1461,"session_id":"e6333dc0-ca22-47ae-bf2f-7d015b5a7fb3"} {"msg":"Crucible stats registered with UUID: 916f8fcf-8aab-4439-b871-4730b03ade1a","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:56.238862285Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1461,"session_id":"e6333dc0-ca22-47ae-bf2f-7d015b5a7fb3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:44:56.238871265Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1461,"session_id":"e6333dc0-ca22-47ae-bf2f-7d015b5a7fb3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:56.239007911Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1461,"":"io task","client":"2","":"downstairs","session_id":"e6333dc0-ca22-47ae-bf2f-7d015b5a7fb3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:56.239038Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1461,"":"io task","client":"0","":"downstairs","session_id":"e6333dc0-ca22-47ae-bf2f-7d015b5a7fb3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:56.23904916Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1461,"":"io task","client":"1","":"downstairs","session_id":"e6333dc0-ca22-47ae-bf2f-7d015b5a7fb3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:56.239195766Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1461,"task":"crutest"} {"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:56.239208765Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1461} {"msg":"916f8fcf-8aab-4439-b871-4730b03ade1a active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:56.239225775Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1461,"session_id":"e6333dc0-ca22-47ae-bf2f-7d015b5a7fb3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:56.239234565Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1461,"client":"0","":"downstairs","session_id":"e6333dc0-ca22-47ae-bf2f-7d015b5a7fb3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:56.239244164Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1461,"client":"1","":"downstairs","session_id":"e6333dc0-ca22-47ae-bf2f-7d015b5a7fb3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:56.239253764Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1461,"client":"2","":"downstairs","session_id":"e6333dc0-ca22-47ae-bf2f-7d015b5a7fb3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:56.239263194Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1461,"":"io task","client":"2","":"downstairs","session_id":"e6333dc0-ca22-47ae-bf2f-7d015b5a7fb3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:56.239273054Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1461,"":"io task","client":"0","":"downstairs","session_id":"e6333dc0-ca22-47ae-bf2f-7d015b5a7fb3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:56.239282873Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1461,"":"io task","client":"1","":"downstairs","session_id":"e6333dc0-ca22-47ae-bf2f-7d015b5a7fb3"} {"msg":"ds_connection connected from Ok(127.0.0.1:55583)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:56.239374751Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1461,"":"io task","client":"1","":"downstairs","session_id":"e6333dc0-ca22-47ae-bf2f-7d015b5a7fb3"} {"msg":"ds_connection connected from Ok(127.0.0.1:34276)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:56.239440619Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1461,"":"io task","client":"0","":"downstairs","session_id":"e6333dc0-ca22-47ae-bf2f-7d015b5a7fb3"} {"msg":"ds_connection connected from Ok(127.0.0.1:61852)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:56.239452398Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1461,"":"io task","client":"2","":"downstairs","session_id":"e6333dc0-ca22-47ae-bf2f-7d015b5a7fb3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:56.240030502Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1461,"client":"0","":"downstairs","session_id":"e6333dc0-ca22-47ae-bf2f-7d015b5a7fb3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:56.240057261Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1461,"client":"2","":"downstairs","session_id":"e6333dc0-ca22-47ae-bf2f-7d015b5a7fb3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:56.240068691Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1461,"client":"1","":"downstairs","session_id":"e6333dc0-ca22-47ae-bf2f-7d015b5a7fb3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:56.240353782Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1461,"session_id":"e6333dc0-ca22-47ae-bf2f-7d015b5a7fb3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:56.240363372Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1461,"session_id":"e6333dc0-ca22-47ae-bf2f-7d015b5a7fb3"} {"msg":"[0]R flush_numbers[0..12]: [119, 127, 120, 131, 123, 121, 131, 110, 127, 129, 118, 126]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:56.240404131Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1461,"":"downstairs","":"downstairs","session_id":"e6333dc0-ca22-47ae-bf2f-7d015b5a7fb3"} {"msg":"[0]R generation[0..12]: [75, 79, 76, 81, 76, 76, 81, 68, 79, 80, 74, 78]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:56.240415071Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1461,"":"downstairs","":"downstairs","session_id":"e6333dc0-ca22-47ae-bf2f-7d015b5a7fb3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:56.24042607Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1461,"":"downstairs","":"downstairs","session_id":"e6333dc0-ca22-47ae-bf2f-7d015b5a7fb3"} {"msg":"[1]R flush_numbers[0..12]: [119, 127, 120, 131, 123, 121, 131, 110, 127, 129, 118, 126]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:56.24043665Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1461,"":"downstairs","":"downstairs","session_id":"e6333dc0-ca22-47ae-bf2f-7d015b5a7fb3"} {"msg":"[1]R generation[0..12]: [75, 79, 76, 81, 76, 76, 81, 68, 79, 80, 74, 78]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:56.24044688Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1461,"":"downstairs","":"downstairs","session_id":"e6333dc0-ca22-47ae-bf2f-7d015b5a7fb3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:56.240457739Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1461,"":"downstairs","":"downstairs","session_id":"e6333dc0-ca22-47ae-bf2f-7d015b5a7fb3"} {"msg":"[2]R flush_numbers[0..12]: [119, 127, 120, 131, 123, 121, 131, 110, 127, 129, 118, 126]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:56.240468369Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1461,"":"downstairs","":"downstairs","session_id":"e6333dc0-ca22-47ae-bf2f-7d015b5a7fb3"} {"msg":"[2]R generation[0..12]: [75, 79, 76, 81, 76, 76, 81, 68, 79, 80, 74, 78]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:56.240478499Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1461,"":"downstairs","":"downstairs","session_id":"e6333dc0-ca22-47ae-bf2f-7d015b5a7fb3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:56.240489348Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1461,"":"downstairs","":"downstairs","session_id":"e6333dc0-ca22-47ae-bf2f-7d015b5a7fb3"} {"msg":"Max found gen is 82","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:56.240499018Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1461,"":"downstairs","":"downstairs","session_id":"e6333dc0-ca22-47ae-bf2f-7d015b5a7fb3"} {"msg":"Generation requested: 82 >= found:82","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:56.240510158Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1461,"":"downstairs","":"downstairs","session_id":"e6333dc0-ca22-47ae-bf2f-7d015b5a7fb3"} {"msg":"Next flush: 132","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:56.240519448Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1461,"":"downstairs","":"downstairs","session_id":"e6333dc0-ca22-47ae-bf2f-7d015b5a7fb3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:56.240528817Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1461,"":"downstairs","":"downstairs","session_id":"e6333dc0-ca22-47ae-bf2f-7d015b5a7fb3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:56.240538437Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1461,"session_id":"e6333dc0-ca22-47ae-bf2f-7d015b5a7fb3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:56.240547107Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1461,"client":"0","":"downstairs","session_id":"e6333dc0-ca22-47ae-bf2f-7d015b5a7fb3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:56.240556837Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1461,"client":"1","":"downstairs","session_id":"e6333dc0-ca22-47ae-bf2f-7d015b5a7fb3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:56.240566286Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1461,"client":"2","":"downstairs","session_id":"e6333dc0-ca22-47ae-bf2f-7d015b5a7fb3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:56.240575946Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1461,"session_id":"e6333dc0-ca22-47ae-bf2f-7d015b5a7fb3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:56.240585076Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1461,"session_id":"e6333dc0-ca22-47ae-bf2f-7d015b5a7fb3"} {"msg":"916f8fcf-8aab-4439-b871-4730b03ade1a is now active with session: e6333dc0-ca22-47ae-bf2f-7d015b5a7fb3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:56.240594835Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1461,"session_id":"e6333dc0-ca22-47ae-bf2f-7d015b5a7fb3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:56.240603455Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1461,"session_id":"e6333dc0-ca22-47ae-bf2f-7d015b5a7fb3"} {"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:56.240612375Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1461} {"msg":"Activated sub_volume 916f8fcf-8aab-4439-b871-4730b03ade1a","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:56.240620275Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1461} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:56.240628294Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1461,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:56.240970975Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1461,"task":"crutest"} 01/30 Write block 193 len 24576 data: 6 7 5 5 5 6 02/30 Flush 03/30 Write block 481 len 16384 data: 10 7 7 7 04/30 Read block 349 len 36864 05/30 Read block 105 len 24576 06/30 Read block 538 len 36864 07/30 Read block 565 len 12288 08/30 Write block 226 len 4096 data: 11 09/30 Write block 331 len 28672 data: 9 8 8 9 9 10 11 10/30 Write block 119 len 4096 data: 8 11/30 Write block 241 len 32768 data: 7 6 5 6 6 8 10 9 12/30 Write block 58 len 20480 data: 9 7 6 9 9 13/30 Write block 580 len 4096 data: 5 14/30 Write block 227 len 16384 data: 11 8 9 9 15/30 Write block 493 len 32768 data: 6 5 7 8 4 6 6 7 16/30 Write block 81 len 32768 data: 6 6 7 7 6 5 4 2 17/30 Read block 94 len 8192 18/30 Write block 118 len 20480 data: 6 9 8 8 8 19/30 Flush 20/30 Write block 142 len 12288 data: 9 8 8 21/30 Write block 470 len 28672 data: 7 5 4 3 5 8 10 22/30 Write block 511 len 36864 data: 11 7 5 5 5 5 6 8 10 23/30 Read block 499 len 20480 24/30 Read block 426 len 36864 25/30 Read block 367 len 8192 26/30 Read block 113 len 40960 27/30 Write block 10 len 28672 data: 7 7 6 6 6 6 7 28/30 Write block 426 len 28672 data: 4 5 4 5 5 4 4 29/30 Write block 22 len 36864 data: 6 4 3 3 2 2 4 5 6 30/30 Read block 309 len 20480 ---------------------------------------------------------------- Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Flush 0 Done Sent Done false 1002 Acked Write 4 Done Sent Done false 1003 Acked Read 9 Done Sent Done false 1004 Acked Read 6 Done Sent Done false 1005 Acked Read 9 Done Sent Done false 1006 Acked Read 3 Done Sent Done false 1007 Acked Write 1 Done Sent Done false 1008 Acked Write 7 Done Sent Done false 1009 Acked Write 1 Done Sent Done false 1010 Acked Write 8 Done Sent Done false 1011 Acked Write 5 Done Sent Done false 1012 Acked Write 1 Done Sent Done false 1013 Acked Write 4 Done Sent Done false 1014 Acked Write 8 Done Sent Done false 1015 Acked Write 8 Done Sent Done false 1016 Acked Read 2 Done Sent Done false 1017 Acked Write 5 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Write 3 Done Sent Done false 1020 Acked Write 7 Done Sent Done false 1021 Acked Write 9 Done Sent Done false 1022 Acked Read 5 Done Sent Done false 1023 Acked Read 9 Done Sent Done false 1024 Acked Read 2 Done Sent Done false 1025 Acked Read 10 Done Sent Done false 1026 Acked Write 7 Done Sent Done false 1027 Acked Write 7 Done Sent Done false 1028 Acked Write 9 Done Sent Done false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 29 1 30 Done 30 1 29 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) None Some(JobId(1018)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:44:56.248892826Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1461} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:44:56.249073361Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1461,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 75 75 75 119 119 119 T F T <--- 1 020-039 79 79 79 127 127 127 T F T <--- 2 040-059 82 76 82 133 120 133 F F F <--- 3 060-079 82 81 82 133 131 133 F F F <--- 4 080-099 82 76 82 133 123 133 F F F <--- 5 100-119 82 76 82 133 121 133 F F F <--- 6 120-139 82 81 82 133 131 133 F F F <--- 7 140-159 68 68 68 110 110 110 T F T <--- 8 160-179 79 79 79 127 127 127 F F F 9 180-199 82 80 82 132 129 132 F T F <--- 10 200-219 74 74 74 118 118 118 F F F 11 220-239 82 78 82 133 126 133 F F F <--- 12 240-259 82 80 82 133 129 133 F F F <--- 13 260-279 76 76 76 120 120 120 F F F 14 280-299 80 80 80 129 129 129 F F F 15 300-319 80 80 80 129 129 129 F F F 16 320-339 82 72 82 133 115 133 F F F <--- 17 340-359 80 80 80 129 129 129 F F F 18 360-379 78 78 78 126 126 126 F F F 19 380-399 78 78 78 126 126 126 F F F 20 400-419 76 76 76 120 120 120 F F F 21 420-439 78 78 78 126 126 126 T F T <--- 22 440-459 76 76 76 121 121 121 F F F 23 460-479 77 77 77 125 125 125 T F T <--- 24 480-499 82 79 82 133 127 133 F F F <--- 25 500-519 82 80 82 133 130 133 T F T <--- 26 520-539 81 81 81 131 131 131 F F F 27 540-559 81 81 81 131 131 131 F F F 28 560-579 80 80 80 129 129 129 F F F 29 580-599 82 79 82 133 127 133 F F F <--- Max gen: 82, Max flush: 133 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 41 Verifying data now No change after verify Loop: 41 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 83 83 83 134 134 134 F F F 1 020-039 83 83 83 134 134 134 F F F 2 040-059 82 82 82 133 133 133 F F F 3 060-079 82 82 82 133 133 133 F F F 4 080-099 82 82 82 133 133 133 F F F 5 100-119 82 82 82 133 133 133 F F F 6 120-139 82 82 82 133 133 133 F F F 7 140-159 83 83 83 134 134 134 F F F 8 160-179 79 79 79 127 127 127 F F F 9 180-199 82 82 82 132 132 132 F F F 10 200-219 74 74 74 118 118 118 F F F 11 220-239 82 82 82 133 133 133 F F F 12 240-259 82 82 82 133 133 133 F F F 13 260-279 76 76 76 120 120 120 F F F 14 280-299 80 80 80 129 129 129 F F F 15 300-319 80 80 80 129 129 129 F F F 16 320-339 82 82 82 133 133 133 F F F 17 340-359 80 80 80 129 129 129 F F F 18 360-379 78 78 78 126 126 126 F F F 19 380-399 78 78 78 126 126 126 F F F 20 400-419 76 76 76 120 120 120 F F F 21 420-439 83 83 83 134 134 134 F F F 22 440-459 76 76 76 121 121 121 F F F 23 460-479 83 83 83 134 134 134 F F F 24 480-499 82 82 82 133 133 133 F F F 25 500-519 83 83 83 134 134 134 F F F 26 520-539 81 81 81 131 131 131 F F F 27 540-559 81 81 81 131 131 131 F F F 28 560-579 80 80 80 129 129 129 F F F 29 580-599 82 82 82 133 133 133 F F F Max gen: 83, Max flush: 134 Begin loop 42 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:00.502294998Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1469,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:45:00.509337315Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1469,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:00.509392274Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1469,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:00.516165758Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1469,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:00.516424041Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1469} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:00.51645263Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1469} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:00.51646108Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1469} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:00.516472589Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1469} {"msg":"Crucible b168e8f6-2767-41b3-9538-1e7643f8a2dd has session id: befa0d3d-2048-49fa-9bb3-035e70610ea7","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:00.51679633Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1469,"session_id":"befa0d3d-2048-49fa-9bb3-035e70610ea7"} {"msg":"Upstairs opts: Upstairs UUID: b168e8f6-2767-41b3-9538-1e7643f8a2dd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:00.516821189Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1469,"session_id":"befa0d3d-2048-49fa-9bb3-035e70610ea7"} {"msg":"Crucible stats registered with UUID: b168e8f6-2767-41b3-9538-1e7643f8a2dd","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:00.516831129Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1469,"session_id":"befa0d3d-2048-49fa-9bb3-035e70610ea7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:45:00.516840089Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1469,"session_id":"befa0d3d-2048-49fa-9bb3-035e70610ea7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:00.516990754Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1469,"":"io task","client":"1","":"downstairs","session_id":"befa0d3d-2048-49fa-9bb3-035e70610ea7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:00.517021853Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1469,"":"io task","client":"0","":"downstairs","session_id":"befa0d3d-2048-49fa-9bb3-035e70610ea7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:00.517032663Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1469,"":"io task","client":"2","":"downstairs","session_id":"befa0d3d-2048-49fa-9bb3-035e70610ea7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:00.51715266Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1469,"task":"crutest"} {"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:00.517167899Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1469} {"msg":"b168e8f6-2767-41b3-9538-1e7643f8a2dd active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:00.517204158Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1469,"session_id":"befa0d3d-2048-49fa-9bb3-035e70610ea7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:00.517213488Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1469,"client":"0","":"downstairs","session_id":"befa0d3d-2048-49fa-9bb3-035e70610ea7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:00.517223188Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1469,"client":"1","":"downstairs","session_id":"befa0d3d-2048-49fa-9bb3-035e70610ea7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:00.517232787Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1469,"client":"2","":"downstairs","session_id":"befa0d3d-2048-49fa-9bb3-035e70610ea7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:00.517242357Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1469,"":"io task","client":"2","":"downstairs","session_id":"befa0d3d-2048-49fa-9bb3-035e70610ea7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:00.517283166Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1469,"":"io task","client":"0","":"downstairs","session_id":"befa0d3d-2048-49fa-9bb3-035e70610ea7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:00.517306055Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1469,"":"io task","client":"1","":"downstairs","session_id":"befa0d3d-2048-49fa-9bb3-035e70610ea7"} {"msg":"ds_connection connected from Ok(127.0.0.1:38285)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:00.517317065Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1469,"":"io task","client":"2","":"downstairs","session_id":"befa0d3d-2048-49fa-9bb3-035e70610ea7"} {"msg":"ds_connection connected from Ok(127.0.0.1:45447)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:00.517327545Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1469,"":"io task","client":"1","":"downstairs","session_id":"befa0d3d-2048-49fa-9bb3-035e70610ea7"} {"msg":"ds_connection connected from Ok(127.0.0.1:33548)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:00.517337724Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1469,"":"io task","client":"0","":"downstairs","session_id":"befa0d3d-2048-49fa-9bb3-035e70610ea7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:00.517996915Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1469,"client":"1","":"downstairs","session_id":"befa0d3d-2048-49fa-9bb3-035e70610ea7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:00.518023675Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1469,"client":"2","":"downstairs","session_id":"befa0d3d-2048-49fa-9bb3-035e70610ea7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:00.518035384Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1469,"client":"0","":"downstairs","session_id":"befa0d3d-2048-49fa-9bb3-035e70610ea7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:00.518367685Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1469,"session_id":"befa0d3d-2048-49fa-9bb3-035e70610ea7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:00.518381564Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1469,"session_id":"befa0d3d-2048-49fa-9bb3-035e70610ea7"} {"msg":"[0]R flush_numbers[0..12]: [134, 134, 133, 133, 133, 133, 133, 134, 127, 132, 118, 133]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:00.518424813Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1469,"":"downstairs","":"downstairs","session_id":"befa0d3d-2048-49fa-9bb3-035e70610ea7"} {"msg":"[0]R generation[0..12]: [83, 83, 82, 82, 82, 82, 82, 83, 79, 82, 74, 82]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:00.518435873Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1469,"":"downstairs","":"downstairs","session_id":"befa0d3d-2048-49fa-9bb3-035e70610ea7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:00.518446762Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1469,"":"downstairs","":"downstairs","session_id":"befa0d3d-2048-49fa-9bb3-035e70610ea7"} {"msg":"[1]R flush_numbers[0..12]: [134, 134, 133, 133, 133, 133, 133, 134, 127, 132, 118, 133]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:00.518457262Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1469,"":"downstairs","":"downstairs","session_id":"befa0d3d-2048-49fa-9bb3-035e70610ea7"} {"msg":"[1]R generation[0..12]: [83, 83, 82, 82, 82, 82, 82, 83, 79, 82, 74, 82]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:00.518467462Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1469,"":"downstairs","":"downstairs","session_id":"befa0d3d-2048-49fa-9bb3-035e70610ea7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:00.518478401Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1469,"":"downstairs","":"downstairs","session_id":"befa0d3d-2048-49fa-9bb3-035e70610ea7"} {"msg":"[2]R flush_numbers[0..12]: [134, 134, 133, 133, 133, 133, 133, 134, 127, 132, 118, 133]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:00.518488781Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1469,"":"downstairs","":"downstairs","session_id":"befa0d3d-2048-49fa-9bb3-035e70610ea7"} {"msg":"[2]R generation[0..12]: [83, 83, 82, 82, 82, 82, 82, 83, 79, 82, 74, 82]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:00.518498981Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1469,"":"downstairs","":"downstairs","session_id":"befa0d3d-2048-49fa-9bb3-035e70610ea7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:00.518509651Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1469,"":"downstairs","":"downstairs","session_id":"befa0d3d-2048-49fa-9bb3-035e70610ea7"} {"msg":"Max found gen is 84","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:00.51851902Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1469,"":"downstairs","":"downstairs","session_id":"befa0d3d-2048-49fa-9bb3-035e70610ea7"} {"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:00.51852872Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1469,"":"downstairs","":"downstairs","session_id":"befa0d3d-2048-49fa-9bb3-035e70610ea7"} {"msg":"Next flush: 135","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:00.51853791Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1469,"":"downstairs","":"downstairs","session_id":"befa0d3d-2048-49fa-9bb3-035e70610ea7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:00.518547209Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1469,"":"downstairs","":"downstairs","session_id":"befa0d3d-2048-49fa-9bb3-035e70610ea7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:00.518558499Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1469,"session_id":"befa0d3d-2048-49fa-9bb3-035e70610ea7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:00.518567149Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1469,"client":"0","":"downstairs","session_id":"befa0d3d-2048-49fa-9bb3-035e70610ea7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:00.518576689Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1469,"client":"1","":"downstairs","session_id":"befa0d3d-2048-49fa-9bb3-035e70610ea7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:00.518586218Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1469,"client":"2","":"downstairs","session_id":"befa0d3d-2048-49fa-9bb3-035e70610ea7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:00.518595878Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1469,"session_id":"befa0d3d-2048-49fa-9bb3-035e70610ea7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:00.518604898Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1469,"session_id":"befa0d3d-2048-49fa-9bb3-035e70610ea7"} {"msg":"b168e8f6-2767-41b3-9538-1e7643f8a2dd is now active with session: befa0d3d-2048-49fa-9bb3-035e70610ea7","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:00.518614758Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1469,"session_id":"befa0d3d-2048-49fa-9bb3-035e70610ea7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:00.518623397Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1469,"session_id":"befa0d3d-2048-49fa-9bb3-035e70610ea7"} {"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:00.518632257Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1469} {"msg":"Activated sub_volume b168e8f6-2767-41b3-9538-1e7643f8a2dd","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:00.518640077Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1469} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:00.518648097Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1469,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:00.518924939Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1469,"task":"crutest"} 01/30 Write block 369 len 24576 data: 7 5 4 4 4 3 02/30 Read block 350 len 40960 03/30 Read block 103 len 4096 04/30 Read block 217 len 12288 05/30 Write block 562 len 36864 data: 10 8 9 9 8 5 7 7 7 06/30 Write block 454 len 36864 data: 4 3 5 6 5 3 2 2 2 07/30 Write block 413 len 40960 data: 9 7 4 5 5 6 7 7 7 7 08/30 Write block 346 len 20480 data: 7 7 6 6 4 09/30 Flush 10/30 Flush 11/30 Write block 196 len 40960 data: 6 6 7 7 5 11 11 10 12 10 12/30 Read block 476 len 40960 13/30 Write block 147 len 32768 data: 7 6 4 4 4 4 3 3 14/30 Write block 120 len 8192 data: 9 9 15/30 Read block 307 len 28672 16/30 Read block 355 len 24576 17/30 Write block 529 len 24576 data: 5 6 6 11 10 10 18/30 Write block 20 len 24576 data: 4 3 7 5 4 4 19/30 Flush 20/30 Write block 3 len 32768 data: 5 7 9 12 10 10 8 8 21/30 Write block 201 len 40960 data: 12 12 11 13 11 10 8 10 10 10 22/30 Read block 579 len 40960 23/30 Read block 17 len 16384 24/30 Read block 120 len 16384 25/30 Flush 26/30 Flush 27/30 Write block 521 len 16384 data: 12 12 11 10 28/30 Read block 558 len 20480 29/30 Write block 219 len 24576 data: 8 8 7 9 10 11 30/30 Write block 69 len 20480 data: 4 5 6 6 7 ---------------------------------------------------------------- Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Read 10 Done Done Done false 1002 Acked Read 1 Sent Done Done false 1003 Acked Read 3 Sent Done Done false 1004 Acked Write 9 Sent Done Done false 1005 Acked Write 9 Sent Done Done false 1006 Acked Write 10 Sent Done Done false 1007 Acked Write 5 Sent Done Done false 1008 Acked Flush 0 Sent Done Done false 1009 Acked Flush 0 Sent Done Done false 1010 Acked Write 10 Sent Done Done false 1011 Acked Read 10 Sent Done Done false 1012 Acked Write 8 Sent Done Done false 1013 Acked Write 2 Sent Done Done false 1014 Acked Read 7 Sent Done Done false 1015 Acked Read 6 Sent Done Done false 1016 Acked Write 6 Sent Done Done false 1017 Acked Write 6 Sent Done Done false 1018 Acked Flush 0 Sent Done Done false 1019 Acked Write 8 Sent Done Done false 1020 Acked Write 10 Sent Done Done false 1021 Acked Read 10 Sent Done Done false 1022 Acked Read 4 Sent Done Done false 1023 Acked Read 4 Sent Done Done false 1024 Acked Flush 0 Sent Done Done false 1025 Acked Flush 0 Sent Done Done false 1026 Acked Write 4 Sent Done Done false 1027 Acked Read 5 Sent Done Done false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 2 2 32 Done 2 28 28 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:45:00.528017856Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1469} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:00.528189651Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1469,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 83 84 84 134 138 138 F F F <--- 1 020-039 83 84 84 134 137 137 F F F <--- 2 040-059 82 82 82 133 133 133 F F F 3 060-079 82 82 82 133 133 133 F T T <--- 4 080-099 82 82 82 133 133 133 F F F 5 100-119 82 82 82 133 133 133 F F F 6 120-139 82 84 84 133 137 137 F F F <--- 7 140-159 83 84 84 134 137 137 F F F <--- 8 160-179 79 79 79 127 127 127 F F F 9 180-199 82 84 84 132 137 137 F F F <--- 10 200-219 74 84 84 118 138 138 F T T <--- 11 220-239 82 82 82 133 133 133 F T T <--- 12 240-259 82 82 82 133 133 133 F F F 13 260-279 76 76 76 120 120 120 F F F 14 280-299 80 80 80 129 129 129 F F F 15 300-319 80 80 80 129 129 129 F F F 16 320-339 82 82 82 133 133 133 F F F 17 340-359 80 84 84 129 135 135 F F F <--- 18 360-379 78 84 84 126 135 135 T F F <--- 19 380-399 78 78 78 126 126 126 F F F 20 400-419 76 84 84 120 135 135 F F F <--- 21 420-439 83 84 84 134 135 135 F F F <--- 22 440-459 76 84 84 121 135 135 F F F <--- 23 460-479 83 84 84 134 135 135 F F F <--- 24 480-499 82 82 82 133 133 133 F F F 25 500-519 83 83 83 134 134 134 F F F 26 520-539 81 84 84 131 137 137 F T T <--- 27 540-559 81 81 81 131 131 131 F F F 28 560-579 80 84 84 129 135 135 F F F <--- 29 580-599 82 82 82 133 133 133 F F F Max gen: 84, Max flush: 138 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 42 Verifying data now No change after verify Loop: 42 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 84 84 84 138 138 138 F F F 1 020-039 84 84 84 137 137 137 F F F 2 040-059 82 82 82 133 133 133 F F F 3 060-079 85 85 85 139 139 139 F F F 4 080-099 82 82 82 133 133 133 F F F 5 100-119 82 82 82 133 133 133 F F F 6 120-139 84 84 84 137 137 137 F F F 7 140-159 84 84 84 137 137 137 F F F 8 160-179 79 79 79 127 127 127 F F F 9 180-199 84 84 84 137 137 137 F F F 10 200-219 85 85 85 139 139 139 F F F 11 220-239 85 85 85 139 139 139 F F F 12 240-259 82 82 82 133 133 133 F F F 13 260-279 76 76 76 120 120 120 F F F 14 280-299 80 80 80 129 129 129 F F F 15 300-319 80 80 80 129 129 129 F F F 16 320-339 82 82 82 133 133 133 F F F 17 340-359 84 84 84 135 135 135 F F F 18 360-379 84 84 84 135 135 135 F F F 19 380-399 78 78 78 126 126 126 F F F 20 400-419 84 84 84 135 135 135 F F F 21 420-439 84 84 84 135 135 135 F F F 22 440-459 84 84 84 135 135 135 F F F 23 460-479 84 84 84 135 135 135 F F F 24 480-499 82 82 82 133 133 133 F F F 25 500-519 83 83 83 134 134 134 F F F 26 520-539 85 85 85 139 139 139 F F F 27 540-559 81 81 81 131 131 131 F F F 28 560-579 84 84 84 135 135 135 F F F 29 580-599 82 82 82 133 133 133 F F F Max gen: 85, Max flush: 139 Begin loop 43 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:04.787347408Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1477,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:04.795461663Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1477,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:04.795753075Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1477} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:04.795789944Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1477} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:04.795801024Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1477} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:04.795812723Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1477} {"msg":"Crucible 63fee47c-b7bb-4db3-99cf-32d4fd0b781b has session id: 294406bd-cb94-4f21-8078-0c8ccffacd8a","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:04.796183433Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1477,"session_id":"294406bd-cb94-4f21-8078-0c8ccffacd8a"} {"msg":"Upstairs opts: Upstairs UUID: 63fee47c-b7bb-4db3-99cf-32d4fd0b781b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:04.796230371Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1477,"session_id":"294406bd-cb94-4f21-8078-0c8ccffacd8a"} {"msg":"Crucible stats registered with UUID: 63fee47c-b7bb-4db3-99cf-32d4fd0b781b","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:04.796240751Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1477,"session_id":"294406bd-cb94-4f21-8078-0c8ccffacd8a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:45:04.796249681Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1477,"session_id":"294406bd-cb94-4f21-8078-0c8ccffacd8a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:04.796393337Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1477,"":"io task","client":"0","":"downstairs","session_id":"294406bd-cb94-4f21-8078-0c8ccffacd8a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:04.796416066Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1477,"":"io task","client":"1","":"downstairs","session_id":"294406bd-cb94-4f21-8078-0c8ccffacd8a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:04.796426346Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1477,"":"io task","client":"2","":"downstairs","session_id":"294406bd-cb94-4f21-8078-0c8ccffacd8a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:04.796586351Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1477,"task":"crutest"} {"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:04.796597951Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1477} {"msg":"63fee47c-b7bb-4db3-99cf-32d4fd0b781b active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:04.79661445Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1477,"session_id":"294406bd-cb94-4f21-8078-0c8ccffacd8a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:04.79662339Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1477,"client":"0","":"downstairs","session_id":"294406bd-cb94-4f21-8078-0c8ccffacd8a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:04.79663303Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1477,"client":"1","":"downstairs","session_id":"294406bd-cb94-4f21-8078-0c8ccffacd8a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:04.796642479Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1477,"client":"2","":"downstairs","session_id":"294406bd-cb94-4f21-8078-0c8ccffacd8a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:04.796653979Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1477,"":"io task","client":"2","":"downstairs","session_id":"294406bd-cb94-4f21-8078-0c8ccffacd8a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:04.796663839Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1477,"":"io task","client":"0","":"downstairs","session_id":"294406bd-cb94-4f21-8078-0c8ccffacd8a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:04.796673919Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1477,"":"io task","client":"1","":"downstairs","session_id":"294406bd-cb94-4f21-8078-0c8ccffacd8a"} {"msg":"ds_connection connected from Ok(127.0.0.1:42213)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:04.796733057Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1477,"":"io task","client":"1","":"downstairs","session_id":"294406bd-cb94-4f21-8078-0c8ccffacd8a"} {"msg":"ds_connection connected from Ok(127.0.0.1:35385)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:04.796743796Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1477,"":"io task","client":"2","":"downstairs","session_id":"294406bd-cb94-4f21-8078-0c8ccffacd8a"} {"msg":"ds_connection connected from Ok(127.0.0.1:59652)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:04.796753896Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1477,"":"io task","client":"0","":"downstairs","session_id":"294406bd-cb94-4f21-8078-0c8ccffacd8a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:04.797403287Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1477,"client":"2","":"downstairs","session_id":"294406bd-cb94-4f21-8078-0c8ccffacd8a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:04.797422487Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1477,"client":"1","":"downstairs","session_id":"294406bd-cb94-4f21-8078-0c8ccffacd8a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:04.797471635Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1477,"client":"0","":"downstairs","session_id":"294406bd-cb94-4f21-8078-0c8ccffacd8a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:04.797741478Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1477,"session_id":"294406bd-cb94-4f21-8078-0c8ccffacd8a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:04.797774527Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1477,"session_id":"294406bd-cb94-4f21-8078-0c8ccffacd8a"} {"msg":"[0]R flush_numbers[0..12]: [138, 137, 133, 139, 133, 133, 137, 137, 127, 137, 139, 139]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:04.797845535Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1477,"":"downstairs","":"downstairs","session_id":"294406bd-cb94-4f21-8078-0c8ccffacd8a"} {"msg":"[0]R generation[0..12]: [84, 84, 82, 85, 82, 82, 84, 84, 79, 84, 85, 85]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:04.797856394Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1477,"":"downstairs","":"downstairs","session_id":"294406bd-cb94-4f21-8078-0c8ccffacd8a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:04.797867354Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1477,"":"downstairs","":"downstairs","session_id":"294406bd-cb94-4f21-8078-0c8ccffacd8a"} {"msg":"[1]R flush_numbers[0..12]: [138, 137, 133, 139, 133, 133, 137, 137, 127, 137, 139, 139]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:04.797877784Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1477,"":"downstairs","":"downstairs","session_id":"294406bd-cb94-4f21-8078-0c8ccffacd8a"} {"msg":"[1]R generation[0..12]: [84, 84, 82, 85, 82, 82, 84, 84, 79, 84, 85, 85]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:04.797889993Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1477,"":"downstairs","":"downstairs","session_id":"294406bd-cb94-4f21-8078-0c8ccffacd8a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:04.797900953Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1477,"":"downstairs","":"downstairs","session_id":"294406bd-cb94-4f21-8078-0c8ccffacd8a"} {"msg":"[2]R flush_numbers[0..12]: [138, 137, 133, 139, 133, 133, 137, 137, 127, 137, 139, 139]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:04.797911353Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1477,"":"downstairs","":"downstairs","session_id":"294406bd-cb94-4f21-8078-0c8ccffacd8a"} {"msg":"[2]R generation[0..12]: [84, 84, 82, 85, 82, 82, 84, 84, 79, 84, 85, 85]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:04.797921583Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1477,"":"downstairs","":"downstairs","session_id":"294406bd-cb94-4f21-8078-0c8ccffacd8a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:04.797932312Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1477,"":"downstairs","":"downstairs","session_id":"294406bd-cb94-4f21-8078-0c8ccffacd8a"} {"msg":"Max found gen is 86","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:04.797941652Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1477,"":"downstairs","":"downstairs","session_id":"294406bd-cb94-4f21-8078-0c8ccffacd8a"} {"msg":"Generation requested: 86 >= found:86","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:04.797951192Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1477,"":"downstairs","":"downstairs","session_id":"294406bd-cb94-4f21-8078-0c8ccffacd8a"} {"msg":"Next flush: 140","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:04.797960341Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1477,"":"downstairs","":"downstairs","session_id":"294406bd-cb94-4f21-8078-0c8ccffacd8a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:04.797969681Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1477,"":"downstairs","":"downstairs","session_id":"294406bd-cb94-4f21-8078-0c8ccffacd8a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:04.797979601Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1477,"session_id":"294406bd-cb94-4f21-8078-0c8ccffacd8a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:04.797988361Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1477,"client":"0","":"downstairs","session_id":"294406bd-cb94-4f21-8078-0c8ccffacd8a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:04.79799807Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1477,"client":"1","":"downstairs","session_id":"294406bd-cb94-4f21-8078-0c8ccffacd8a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:04.79800777Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1477,"client":"2","":"downstairs","session_id":"294406bd-cb94-4f21-8078-0c8ccffacd8a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:04.79801748Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1477,"session_id":"294406bd-cb94-4f21-8078-0c8ccffacd8a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:04.798026389Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1477,"session_id":"294406bd-cb94-4f21-8078-0c8ccffacd8a"} {"msg":"63fee47c-b7bb-4db3-99cf-32d4fd0b781b is now active with session: 294406bd-cb94-4f21-8078-0c8ccffacd8a","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:04.798037459Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1477,"session_id":"294406bd-cb94-4f21-8078-0c8ccffacd8a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:04.798046099Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1477,"session_id":"294406bd-cb94-4f21-8078-0c8ccffacd8a"} {"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:04.798055189Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1477} {"msg":"Activated sub_volume 63fee47c-b7bb-4db3-99cf-32d4fd0b781b","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:04.798063278Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1477} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:04.798071388Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1477,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:04.798380899Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1477,"task":"crutest"} 01/30 Write block 202 len 16384 data: 13 12 14 12 02/30 Write block 234 len 16384 data: 9 6 5 4 03/30 Read block 327 len 32768 04/30 Read block 457 len 36864 05/30 Read block 121 len 28672 06/30 Read block 223 len 32768 07/30 Read block 442 len 20480 08/30 Read block 542 len 8192 09/30 Write block 45 len 32768 data: 4 3 4 5 4 5 5 5 10/30 Read block 266 len 8192 11/30 Read block 213 len 28672 12/30 Read block 347 len 32768 13/30 Read block 273 len 40960 14/30 Read block 37 len 20480 15/30 Write block 423 len 36864 data: 6 6 5 5 6 5 6 6 5 16/30 Read block 204 len 20480 17/30 Write block 364 len 4096 data: 12 18/30 Write block 560 len 32768 data: 9 9 11 9 10 10 9 6 19/30 Read block 29 len 8192 20/30 Read block 144 len 40960 21/30 Flush 22/30 Write block 563 len 8192 data: 10 11 23/30 Read block 359 len 24576 24/30 Flush 25/30 Write block 581 len 4096 data: 6 26/30 Read block 526 len 28672 27/30 Write block 165 len 16384 data: 8 9 8 9 28/30 Read block 343 len 16384 29/30 Read block 5 len 28672 30/30 Write block 571 len 40960 data: 9 6 7 7 6 8 7 8 6 6 ---------------------------------------------------------------- Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Write 4 Done Done Done false 1002 Acked Read 8 Done Done Done false 1003 Acked Read 9 Done Done Done false 1004 Acked Read 7 Done Done Done false 1005 Acked Read 8 Done Done Done false 1006 Acked Read 5 Done Sent Done false 1007 Acked Read 2 Done Sent Done false 1008 Acked Write 8 Done Sent Done false 1009 Acked Read 2 Done Sent Done false 1010 Acked Read 7 Done Sent Done false 1011 Acked Read 8 Done Sent Done false 1012 Acked Read 10 Done Sent Done false 1013 Acked Read 5 Done Sent Done false 1014 Acked Write 9 Done Sent Done false 1015 Acked Read 5 Done Sent Done false 1016 Acked Write 1 Done Sent Done false 1017 Acked Write 8 Done Sent Done false 1018 Acked Read 2 Done Sent Done false 1019 Acked Read 10 Done Sent Done false 1020 Acked Flush 0 Done Sent Done false 1021 Acked Write 2 Done Sent Done false 1022 Acked Read 6 Done Sent Done false 1023 Acked Flush 0 Done Sent Done false 1024 Acked Write 1 Done Sent Done false 1025 Acked Read 7 Done Sent Done false 1026 Acked Write 4 Done Sent Done false 1027 Acked Read 4 Done Sent Done false 1028 Acked Read 7 Done Sent Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 24 1 26 Done 29 6 29 64 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) None Some(JobId(1023)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:45:04.807542775Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1477} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:04.80772787Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1477,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 84 84 84 138 138 138 F F F 1 020-039 84 84 84 137 137 137 F F F 2 040-059 86 82 86 140 133 140 F F F <--- 3 060-079 85 85 85 139 139 139 F F F 4 080-099 82 82 82 133 133 133 F F F 5 100-119 82 82 82 133 133 133 F F F 6 120-139 84 84 84 137 137 137 F F F 7 140-159 84 84 84 137 137 137 F F F 8 160-179 79 79 79 127 127 127 T F T <--- 9 180-199 84 84 84 137 137 137 F F F 10 200-219 86 85 86 140 139 140 F T F <--- 11 220-239 86 85 86 140 139 140 F T F <--- 12 240-259 82 82 82 133 133 133 F F F 13 260-279 76 76 76 120 120 120 F F F 14 280-299 80 80 80 129 129 129 F F F 15 300-319 80 80 80 129 129 129 F F F 16 320-339 82 82 82 133 133 133 F F F 17 340-359 84 84 84 135 135 135 F F F 18 360-379 86 84 86 140 135 140 F F F <--- 19 380-399 78 78 78 126 126 126 F F F 20 400-419 84 84 84 135 135 135 F F F 21 420-439 86 84 86 140 135 140 F F F <--- 22 440-459 84 84 84 135 135 135 F F F 23 460-479 84 84 84 135 135 135 F F F 24 480-499 82 82 82 133 133 133 F F F 25 500-519 83 83 83 134 134 134 F F F 26 520-539 85 85 85 139 139 139 F F F 27 540-559 81 81 81 131 131 131 F F F 28 560-579 86 84 86 141 135 141 T F T <--- 29 580-599 82 82 82 133 133 133 T F T <--- Max gen: 86, Max flush: 141 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 43 Verifying data now No change after verify Loop: 43 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 84 84 84 138 138 138 F F F 1 020-039 84 84 84 137 137 137 F F F 2 040-059 86 86 86 140 140 140 F F F 3 060-079 85 85 85 139 139 139 F F F 4 080-099 82 82 82 133 133 133 F F F 5 100-119 82 82 82 133 133 133 F F F 6 120-139 84 84 84 137 137 137 F F F 7 140-159 84 84 84 137 137 137 F F F 8 160-179 87 87 87 142 142 142 F F F 9 180-199 84 84 84 137 137 137 F F F 10 200-219 86 86 86 140 140 140 F F F 11 220-239 86 86 86 140 140 140 F F F 12 240-259 82 82 82 133 133 133 F F F 13 260-279 76 76 76 120 120 120 F F F 14 280-299 80 80 80 129 129 129 F F F 15 300-319 80 80 80 129 129 129 F F F 16 320-339 82 82 82 133 133 133 F F F 17 340-359 84 84 84 135 135 135 F F F 18 360-379 86 86 86 140 140 140 F F F 19 380-399 78 78 78 126 126 126 F F F 20 400-419 84 84 84 135 135 135 F F F 21 420-439 86 86 86 140 140 140 F F F 22 440-459 84 84 84 135 135 135 F F F 23 460-479 84 84 84 135 135 135 F F F 24 480-499 82 82 82 133 133 133 F F F 25 500-519 83 83 83 134 134 134 F F F 26 520-539 85 85 85 139 139 139 F F F 27 540-559 81 81 81 131 131 131 F F F 28 560-579 87 87 87 142 142 142 F F F 29 580-599 87 87 87 142 142 142 F F F Max gen: 87, Max flush: 142 Begin loop 44 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:09.049324282Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1485,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:45:09.05633107Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1485,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:09.056386518Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1485,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:09.063199142Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1485,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:09.063461784Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1485} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:09.063486883Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1485} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:09.063495343Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1485} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:09.063506853Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1485} {"msg":"Crucible 5e88159a-8a50-4a5f-85ac-184da71ee190 has session id: 2b316514-8c53-4726-8967-f91a18409837","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:09.063906511Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1485,"session_id":"2b316514-8c53-4726-8967-f91a18409837"} {"msg":"Upstairs opts: Upstairs UUID: 5e88159a-8a50-4a5f-85ac-184da71ee190, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:09.063928051Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1485,"session_id":"2b316514-8c53-4726-8967-f91a18409837"} {"msg":"Crucible stats registered with UUID: 5e88159a-8a50-4a5f-85ac-184da71ee190","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:09.06393801Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1485,"session_id":"2b316514-8c53-4726-8967-f91a18409837"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:45:09.06394721Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1485,"session_id":"2b316514-8c53-4726-8967-f91a18409837"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:09.064081296Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1485,"":"io task","client":"1","":"downstairs","session_id":"2b316514-8c53-4726-8967-f91a18409837"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:09.064098246Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1485,"":"io task","client":"0","":"downstairs","session_id":"2b316514-8c53-4726-8967-f91a18409837"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:09.064108255Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1485,"":"io task","client":"2","":"downstairs","session_id":"2b316514-8c53-4726-8967-f91a18409837"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:09.064277541Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1485,"task":"crutest"} {"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:09.06429414Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1485} {"msg":"5e88159a-8a50-4a5f-85ac-184da71ee190 active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:09.064328299Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1485,"session_id":"2b316514-8c53-4726-8967-f91a18409837"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:09.064337679Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1485,"client":"0","":"downstairs","session_id":"2b316514-8c53-4726-8967-f91a18409837"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:09.064347319Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1485,"client":"1","":"downstairs","session_id":"2b316514-8c53-4726-8967-f91a18409837"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:09.064356968Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1485,"client":"2","":"downstairs","session_id":"2b316514-8c53-4726-8967-f91a18409837"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:09.064366458Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1485,"":"io task","client":"2","":"downstairs","session_id":"2b316514-8c53-4726-8967-f91a18409837"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:09.064376428Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1485,"":"io task","client":"0","":"downstairs","session_id":"2b316514-8c53-4726-8967-f91a18409837"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:09.064386247Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1485,"":"io task","client":"1","":"downstairs","session_id":"2b316514-8c53-4726-8967-f91a18409837"} {"msg":"ds_connection connected from Ok(127.0.0.1:57347)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:09.064420296Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1485,"":"io task","client":"2","":"downstairs","session_id":"2b316514-8c53-4726-8967-f91a18409837"} {"msg":"ds_connection connected from Ok(127.0.0.1:51531)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:09.064497114Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1485,"":"io task","client":"0","":"downstairs","session_id":"2b316514-8c53-4726-8967-f91a18409837"} {"msg":"ds_connection connected from Ok(127.0.0.1:48547)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:09.064543853Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1485,"":"io task","client":"1","":"downstairs","session_id":"2b316514-8c53-4726-8967-f91a18409837"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:09.065143306Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1485,"client":"1","":"downstairs","session_id":"2b316514-8c53-4726-8967-f91a18409837"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:09.065169985Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1485,"client":"0","":"downstairs","session_id":"2b316514-8c53-4726-8967-f91a18409837"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:09.065266212Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1485,"client":"2","":"downstairs","session_id":"2b316514-8c53-4726-8967-f91a18409837"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:09.065454727Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1485,"session_id":"2b316514-8c53-4726-8967-f91a18409837"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:09.065464346Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1485,"session_id":"2b316514-8c53-4726-8967-f91a18409837"} {"msg":"[0]R flush_numbers[0..12]: [138, 137, 140, 139, 133, 133, 137, 137, 142, 137, 140, 140]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:09.065569183Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1485,"":"downstairs","":"downstairs","session_id":"2b316514-8c53-4726-8967-f91a18409837"} {"msg":"[0]R generation[0..12]: [84, 84, 86, 85, 82, 82, 84, 84, 87, 84, 86, 86]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:09.065580233Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1485,"":"downstairs","":"downstairs","session_id":"2b316514-8c53-4726-8967-f91a18409837"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:09.065591093Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1485,"":"downstairs","":"downstairs","session_id":"2b316514-8c53-4726-8967-f91a18409837"} {"msg":"[1]R flush_numbers[0..12]: [138, 137, 140, 139, 133, 133, 137, 137, 142, 137, 140, 140]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:09.065601662Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1485,"":"downstairs","":"downstairs","session_id":"2b316514-8c53-4726-8967-f91a18409837"} {"msg":"[1]R generation[0..12]: [84, 84, 86, 85, 82, 82, 84, 84, 87, 84, 86, 86]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:09.065611872Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1485,"":"downstairs","":"downstairs","session_id":"2b316514-8c53-4726-8967-f91a18409837"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:09.065622762Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1485,"":"downstairs","":"downstairs","session_id":"2b316514-8c53-4726-8967-f91a18409837"} {"msg":"[2]R flush_numbers[0..12]: [138, 137, 140, 139, 133, 133, 137, 137, 142, 137, 140, 140]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:09.065633341Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1485,"":"downstairs","":"downstairs","session_id":"2b316514-8c53-4726-8967-f91a18409837"} {"msg":"[2]R generation[0..12]: [84, 84, 86, 85, 82, 82, 84, 84, 87, 84, 86, 86]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:09.065646381Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1485,"":"downstairs","":"downstairs","session_id":"2b316514-8c53-4726-8967-f91a18409837"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:09.065657401Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1485,"":"downstairs","":"downstairs","session_id":"2b316514-8c53-4726-8967-f91a18409837"} {"msg":"Max found gen is 88","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:09.06566684Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1485,"":"downstairs","":"downstairs","session_id":"2b316514-8c53-4726-8967-f91a18409837"} {"msg":"Generation requested: 88 >= found:88","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:09.06567676Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1485,"":"downstairs","":"downstairs","session_id":"2b316514-8c53-4726-8967-f91a18409837"} {"msg":"Next flush: 143","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:09.06568611Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1485,"":"downstairs","":"downstairs","session_id":"2b316514-8c53-4726-8967-f91a18409837"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:09.0656955Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1485,"":"downstairs","":"downstairs","session_id":"2b316514-8c53-4726-8967-f91a18409837"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:09.065705689Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1485,"session_id":"2b316514-8c53-4726-8967-f91a18409837"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:09.065714379Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1485,"client":"0","":"downstairs","session_id":"2b316514-8c53-4726-8967-f91a18409837"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:09.065723759Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1485,"client":"1","":"downstairs","session_id":"2b316514-8c53-4726-8967-f91a18409837"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:09.065733379Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1485,"client":"2","":"downstairs","session_id":"2b316514-8c53-4726-8967-f91a18409837"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:09.065743138Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1485,"session_id":"2b316514-8c53-4726-8967-f91a18409837"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:09.065752168Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1485,"session_id":"2b316514-8c53-4726-8967-f91a18409837"} {"msg":"5e88159a-8a50-4a5f-85ac-184da71ee190 is now active with session: 2b316514-8c53-4726-8967-f91a18409837","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:09.065761988Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1485,"session_id":"2b316514-8c53-4726-8967-f91a18409837"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:09.065770557Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1485,"session_id":"2b316514-8c53-4726-8967-f91a18409837"} {"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:09.065779617Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1485} {"msg":"Activated sub_volume 5e88159a-8a50-4a5f-85ac-184da71ee190","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:09.065787947Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1485} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:09.065797457Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1485,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:09.066121437Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1485,"task":"crutest"} 01/30 Write block 409 len 20480 data: 7 9 9 9 10 02/30 Write block 122 len 40960 data: 9 9 7 6 6 8 8 6 7 8 03/30 Write block 587 len 8192 data: 10 9 04/30 Write block 507 len 4096 data: 8 05/30 Read block 515 len 36864 06/30 Read block 44 len 32768 07/30 Read block 78 len 36864 08/30 Write block 454 len 28672 data: 5 4 6 7 6 4 3 09/30 Write block 213 len 8192 data: 9 9 10/30 Write block 279 len 40960 data: 6 5 5 5 5 6 8 5 5 5 11/30 Write block 574 len 28672 data: 8 7 9 8 9 7 7 12/30 Read block 39 len 24576 13/30 Write block 429 len 32768 data: 7 7 6 5 4 4 7 7 14/30 Write block 540 len 16384 data: 12 9 7 7 15/30 Write block 367 len 36864 data: 10 8 8 6 5 5 5 4 7 16/30 Write block 265 len 12288 data: 3 3 4 17/30 Write block 33 len 4096 data: 9 18/30 Write block 45 len 24576 data: 5 4 5 6 5 6 19/30 Read block 364 len 4096 20/30 Read block 2 len 24576 21/30 Read block 111 len 16384 22/30 Read block 313 len 40960 23/30 Write block 221 len 8192 data: 8 10 24/30 Read block 291 len 12288 25/30 Read block 284 len 32768 26/30 Write block 455 len 40960 data: 5 7 8 7 5 4 3 3 3 2 27/30 Write block 489 len 28672 data: 8 9 10 8 7 6 8 28/30 Read block 226 len 20480 29/30 Write block 536 len 28672 data: 10 12 13 13 13 10 8 30/30 Write block 194 len 20480 data: 8 6 7 7 8 ---------------------------------------------------------------- Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Write 10 Done Done Done false 1002 Acked Write 2 Sent Done Done false 1003 Acked Write 1 Sent Done Done false 1004 Acked Read 9 Sent Done Done false 1005 Acked Read 8 Sent Done Done false 1006 Acked Read 9 Sent Done Done false 1007 Acked Write 7 Sent Done Done false 1008 Acked Write 2 Sent Done Done false 1009 Acked Write 10 Sent Done Done false 1010 Acked Write 7 Sent Done Done false 1011 Acked Read 6 Sent Done Done false 1012 Acked Write 8 Sent Done Done false 1013 Acked Write 4 Sent Done Done false 1014 Acked Write 9 Sent Done Done false 1015 Acked Write 3 Sent Done Done false 1016 Acked Write 1 Sent Done Done false 1017 Acked Write 6 Sent Done Done false 1018 Acked Read 1 Sent Done Done false 1019 Acked Read 6 Sent Done Done false 1020 Acked Read 4 Sent Done Done false 1021 Acked Read 10 Sent Done Done false 1022 Acked Write 2 Sent Done Done false 1023 Acked Read 3 Sent Done Done false 1024 Acked Read 8 Sent Done Done false 1025 Acked Write 10 Sent Done Done false 1026 Acked Write 7 Sent Done Done false 1027 Acked Read 5 Sent Done Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 2 2 32 Done 2 28 28 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:45:09.073517754Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1485} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:09.073701999Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1485,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 84 84 84 138 138 138 F F F 1 020-039 84 84 84 137 137 137 F T T <--- 2 040-059 86 86 86 140 140 140 F T T <--- 3 060-079 85 85 85 139 139 139 F F F 4 080-099 82 82 82 133 133 133 F F F 5 100-119 82 82 82 133 133 133 F F F 6 120-139 84 84 84 137 137 137 T T T 7 140-159 84 84 84 137 137 137 F F F 8 160-179 87 87 87 142 142 142 F F F 9 180-199 84 84 84 137 137 137 F T T <--- 10 200-219 86 86 86 140 140 140 F T T <--- 11 220-239 86 86 86 140 140 140 F T T <--- 12 240-259 82 82 82 133 133 133 F F F 13 260-279 76 76 76 120 120 120 F T T <--- 14 280-299 80 80 80 129 129 129 F T T <--- 15 300-319 80 80 80 129 129 129 F F F 16 320-339 82 82 82 133 133 133 F F F 17 340-359 84 84 84 135 135 135 F F F 18 360-379 86 86 86 140 140 140 F T T <--- 19 380-399 78 78 78 126 126 126 F F F 20 400-419 84 84 84 135 135 135 T T T 21 420-439 86 86 86 140 140 140 F T T <--- 22 440-459 84 84 84 135 135 135 F T T <--- 23 460-479 84 84 84 135 135 135 F T T <--- 24 480-499 82 82 82 133 133 133 F T T <--- 25 500-519 83 83 83 134 134 134 F T T <--- 26 520-539 85 85 85 139 139 139 F T T <--- 27 540-559 81 81 81 131 131 131 F T T <--- 28 560-579 87 87 87 142 142 142 F T T <--- 29 580-599 87 87 87 142 142 142 F T T <--- Max gen: 87, Max flush: 142 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 44 Verifying data now No change after verify Loop: 44 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 84 84 84 138 138 138 F F F 1 020-039 88 88 88 143 143 143 F F F 2 040-059 88 88 88 143 143 143 F F F 3 060-079 85 85 85 139 139 139 F F F 4 080-099 82 82 82 133 133 133 F F F 5 100-119 82 82 82 133 133 133 F F F 6 120-139 88 88 88 143 143 143 F F F 7 140-159 84 84 84 137 137 137 F F F 8 160-179 87 87 87 142 142 142 F F F 9 180-199 88 88 88 143 143 143 F F F 10 200-219 88 88 88 143 143 143 F F F 11 220-239 88 88 88 143 143 143 F F F 12 240-259 82 82 82 133 133 133 F F F 13 260-279 88 88 88 143 143 143 F F F 14 280-299 88 88 88 143 143 143 F F F 15 300-319 80 80 80 129 129 129 F F F 16 320-339 82 82 82 133 133 133 F F F 17 340-359 84 84 84 135 135 135 F F F 18 360-379 88 88 88 143 143 143 F F F 19 380-399 78 78 78 126 126 126 F F F 20 400-419 88 88 88 143 143 143 F F F 21 420-439 88 88 88 143 143 143 F F F 22 440-459 88 88 88 143 143 143 F F F 23 460-479 88 88 88 143 143 143 F F F 24 480-499 88 88 88 143 143 143 F F F 25 500-519 88 88 88 143 143 143 F F F 26 520-539 88 88 88 143 143 143 F F F 27 540-559 88 88 88 143 143 143 F F F 28 560-579 88 88 88 143 143 143 F F F 29 580-599 88 88 88 143 143 143 F F F Max gen: 88, Max flush: 143 Begin loop 45 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:13.346607558Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1493,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:13.35452789Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1493,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:13.354809322Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1493} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:13.354834501Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1493} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:13.354843271Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1493} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:13.35485474Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1493} {"msg":"Crucible 6feb7a4f-a628-4c06-aa98-ca3d957342af has session id: 832301dd-dd1f-46cc-9a21-354faac0c707","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:13.355154242Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1493,"session_id":"832301dd-dd1f-46cc-9a21-354faac0c707"} {"msg":"Upstairs opts: Upstairs UUID: 6feb7a4f-a628-4c06-aa98-ca3d957342af, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:13.355177571Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1493,"session_id":"832301dd-dd1f-46cc-9a21-354faac0c707"} {"msg":"Crucible stats registered with UUID: 6feb7a4f-a628-4c06-aa98-ca3d957342af","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:13.355187611Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1493,"session_id":"832301dd-dd1f-46cc-9a21-354faac0c707"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:45:13.355196621Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1493,"session_id":"832301dd-dd1f-46cc-9a21-354faac0c707"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:13.355353146Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1493,"":"io task","client":"2","":"downstairs","session_id":"832301dd-dd1f-46cc-9a21-354faac0c707"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:13.355388255Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1493,"":"io task","client":"0","":"downstairs","session_id":"832301dd-dd1f-46cc-9a21-354faac0c707"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:13.355399605Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1493,"":"io task","client":"1","":"downstairs","session_id":"832301dd-dd1f-46cc-9a21-354faac0c707"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:13.355505362Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1493,"task":"crutest"} {"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:13.355519271Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1493} {"msg":"6feb7a4f-a628-4c06-aa98-ca3d957342af active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:13.35555794Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1493,"session_id":"832301dd-dd1f-46cc-9a21-354faac0c707"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:13.3555674Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1493,"client":"0","":"downstairs","session_id":"832301dd-dd1f-46cc-9a21-354faac0c707"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:13.35557737Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1493,"client":"1","":"downstairs","session_id":"832301dd-dd1f-46cc-9a21-354faac0c707"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:13.355586969Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1493,"client":"2","":"downstairs","session_id":"832301dd-dd1f-46cc-9a21-354faac0c707"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:13.355596449Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1493,"":"io task","client":"2","":"downstairs","session_id":"832301dd-dd1f-46cc-9a21-354faac0c707"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:13.355606569Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1493,"":"io task","client":"0","":"downstairs","session_id":"832301dd-dd1f-46cc-9a21-354faac0c707"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:13.355616418Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1493,"":"io task","client":"1","":"downstairs","session_id":"832301dd-dd1f-46cc-9a21-354faac0c707"} {"msg":"ds_connection connected from Ok(127.0.0.1:47654)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:13.355682197Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1493,"":"io task","client":"1","":"downstairs","session_id":"832301dd-dd1f-46cc-9a21-354faac0c707"} {"msg":"ds_connection connected from Ok(127.0.0.1:56560)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:13.355693076Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1493,"":"io task","client":"2","":"downstairs","session_id":"832301dd-dd1f-46cc-9a21-354faac0c707"} {"msg":"ds_connection connected from Ok(127.0.0.1:63933)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:13.355703346Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1493,"":"io task","client":"0","":"downstairs","session_id":"832301dd-dd1f-46cc-9a21-354faac0c707"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:13.356381486Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1493,"client":"1","":"downstairs","session_id":"832301dd-dd1f-46cc-9a21-354faac0c707"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:13.356421365Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1493,"client":"2","":"downstairs","session_id":"832301dd-dd1f-46cc-9a21-354faac0c707"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:13.356434035Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1493,"client":"0","":"downstairs","session_id":"832301dd-dd1f-46cc-9a21-354faac0c707"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:13.356685158Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1493,"session_id":"832301dd-dd1f-46cc-9a21-354faac0c707"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:13.356723667Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1493,"session_id":"832301dd-dd1f-46cc-9a21-354faac0c707"} {"msg":"[0]R flush_numbers[0..12]: [138, 143, 143, 139, 133, 133, 143, 137, 142, 143, 143, 143]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:13.356761335Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1493,"":"downstairs","":"downstairs","session_id":"832301dd-dd1f-46cc-9a21-354faac0c707"} {"msg":"[0]R generation[0..12]: [84, 88, 88, 85, 82, 82, 88, 84, 87, 88, 88, 88]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:13.356772215Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1493,"":"downstairs","":"downstairs","session_id":"832301dd-dd1f-46cc-9a21-354faac0c707"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:13.356783085Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1493,"":"downstairs","":"downstairs","session_id":"832301dd-dd1f-46cc-9a21-354faac0c707"} {"msg":"[1]R flush_numbers[0..12]: [138, 143, 143, 139, 133, 133, 143, 137, 142, 143, 143, 143]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:13.356793565Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1493,"":"downstairs","":"downstairs","session_id":"832301dd-dd1f-46cc-9a21-354faac0c707"} {"msg":"[1]R generation[0..12]: [84, 88, 88, 85, 82, 82, 88, 84, 87, 88, 88, 88]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:13.356803664Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1493,"":"downstairs","":"downstairs","session_id":"832301dd-dd1f-46cc-9a21-354faac0c707"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:13.356814504Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1493,"":"downstairs","":"downstairs","session_id":"832301dd-dd1f-46cc-9a21-354faac0c707"} {"msg":"[2]R flush_numbers[0..12]: [138, 143, 143, 139, 133, 133, 143, 137, 142, 143, 143, 143]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:13.356825024Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1493,"":"downstairs","":"downstairs","session_id":"832301dd-dd1f-46cc-9a21-354faac0c707"} {"msg":"[2]R generation[0..12]: [84, 88, 88, 85, 82, 82, 88, 84, 87, 88, 88, 88]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:13.356835153Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1493,"":"downstairs","":"downstairs","session_id":"832301dd-dd1f-46cc-9a21-354faac0c707"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:13.356845913Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1493,"":"downstairs","":"downstairs","session_id":"832301dd-dd1f-46cc-9a21-354faac0c707"} {"msg":"Max found gen is 89","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:13.356855213Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1493,"":"downstairs","":"downstairs","session_id":"832301dd-dd1f-46cc-9a21-354faac0c707"} {"msg":"Generation requested: 90 >= found:89","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:13.356864852Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1493,"":"downstairs","":"downstairs","session_id":"832301dd-dd1f-46cc-9a21-354faac0c707"} {"msg":"Next flush: 144","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:13.356875442Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1493,"":"downstairs","":"downstairs","session_id":"832301dd-dd1f-46cc-9a21-354faac0c707"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:13.356884752Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1493,"":"downstairs","":"downstairs","session_id":"832301dd-dd1f-46cc-9a21-354faac0c707"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:13.356894622Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1493,"session_id":"832301dd-dd1f-46cc-9a21-354faac0c707"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:13.356903341Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1493,"client":"0","":"downstairs","session_id":"832301dd-dd1f-46cc-9a21-354faac0c707"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:13.356913101Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1493,"client":"1","":"downstairs","session_id":"832301dd-dd1f-46cc-9a21-354faac0c707"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:13.356922871Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1493,"client":"2","":"downstairs","session_id":"832301dd-dd1f-46cc-9a21-354faac0c707"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:13.35693275Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1493,"session_id":"832301dd-dd1f-46cc-9a21-354faac0c707"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:13.35694184Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1493,"session_id":"832301dd-dd1f-46cc-9a21-354faac0c707"} {"msg":"6feb7a4f-a628-4c06-aa98-ca3d957342af is now active with session: 832301dd-dd1f-46cc-9a21-354faac0c707","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:13.35695157Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1493,"session_id":"832301dd-dd1f-46cc-9a21-354faac0c707"} {"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:13.35696061Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1493} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:13.356968089Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1493,"session_id":"832301dd-dd1f-46cc-9a21-354faac0c707"} {"msg":"Activated sub_volume 6feb7a4f-a628-4c06-aa98-ca3d957342af","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:13.356977259Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1493} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:13.356985189Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1493,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:13.35728347Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1493,"task":"crutest"} 01/30 Write block 569 len 16384 data: 8 8 10 7 02/30 Read block 335 len 20480 03/30 Write block 48 len 12288 data: 7 6 7 04/30 Write block 417 len 40960 data: 6 7 8 8 8 8 7 7 6 6 05/30 Write block 182 len 20480 data: 6 6 9 9 10 06/30 Read block 407 len 16384 07/30 Write block 511 len 8192 data: 12 8 08/30 Read block 519 len 4096 09/30 Write block 111 len 8192 data: 4 6 10/30 Flush 11/30 Write block 13 len 12288 data: 7 7 7 12/30 Read block 159 len 36864 13/30 Write block 106 len 40960 data: 2 2 2 4 4 5 7 8 7 7 14/30 Read block 252 len 20480 15/30 Read block 354 len 32768 16/30 Read block 594 len 12288 17/30 Write block 223 len 36864 data: 11 12 12 12 12 9 10 10 11 18/30 Read block 108 len 24576 19/30 Write block 416 len 24576 data: 6 7 8 9 9 9 20/30 Write block 67 len 20480 data: 6 4 5 6 7 21/30 Write block 281 len 4096 data: 6 22/30 Write block 299 len 32768 data: 7 8 8 5 6 6 6 5 23/30 Write block 211 len 12288 data: 8 9 10 24/30 Read block 416 len 16384 25/30 Flush 26/30 Write block 560 len 8192 data: 10 10 27/30 Write block 542 len 8192 data: 9 8 28/30 Write block 538 len 8192 data: 14 14 29/30 Read block 355 len 4096 30/30 Read block 248 len 12288 ---------------------------------------------------------------- Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Read 5 Done Done Sent false 1002 Acked Write 3 Done Done Sent false 1003 Acked Write 10 Done Done Sent false 1004 Acked Write 5 Done Done Sent false 1005 Acked Read 4 Done Done Sent false 1006 Acked Write 2 Done Done Sent false 1007 Acked Read 1 Done Done Sent false 1008 Acked Write 2 Done Done Sent false 1009 Acked Flush 0 Done Done Sent false 1010 Acked Write 3 Done Done Sent false 1011 Acked Read 9 Done Done Sent false 1012 Acked Write 10 Done Done Sent false 1013 Acked Read 5 Done Done Sent false 1014 Acked Read 8 Done Done Sent false 1015 Acked Read 3 Done Done Sent false 1016 Acked Write 9 Done Done Sent false 1017 Acked Read 6 Done Done Sent false 1018 Acked Write 6 Done Done Sent false 1019 Acked Write 5 Done Done Sent false 1020 Acked Write 1 Done Done Sent false 1021 Acked Write 8 Done Done Sent false 1022 Acked Write 3 Done Done Sent false 1023 Acked Read 4 Done Done Sent false 1024 Acked Flush 0 Done Done Sent false 1025 Acked Write 2 Done Done Sent false 1026 Acked Write 2 Done Done Sent false 1027 Acked Write 2 Done Done Sent false 1028 Acked Read 1 Done Done Sent false 1029 Acked Read 3 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 29 29 Done 30 30 1 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:45:13.365405186Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1493} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:13.365574841Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1493,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 90 90 84 145 145 138 F F F <--- 1 020-039 88 88 88 143 143 143 F F F 2 040-059 90 90 88 144 144 143 F F F <--- 3 060-079 90 90 85 145 145 139 F F F <--- 4 080-099 82 82 82 133 133 133 F F F 5 100-119 90 90 82 145 145 133 F F F <--- 6 120-139 88 88 88 143 143 143 F F F 7 140-159 84 84 84 137 137 137 F F F 8 160-179 87 87 87 142 142 142 F F F 9 180-199 90 90 88 144 144 143 F F F <--- 10 200-219 90 90 88 145 145 143 F F F <--- 11 220-239 90 90 88 145 145 143 F F F <--- 12 240-259 82 82 82 133 133 133 F F F 13 260-279 88 88 88 143 143 143 F F F 14 280-299 90 90 88 145 145 143 F F F <--- 15 300-319 90 90 80 145 145 129 F F F <--- 16 320-339 82 82 82 133 133 133 F F F 17 340-359 84 84 84 135 135 135 F F F 18 360-379 88 88 88 143 143 143 F F F 19 380-399 78 78 78 126 126 126 F F F 20 400-419 90 90 88 145 145 143 F F F <--- 21 420-439 90 90 88 145 145 143 F F F <--- 22 440-459 88 88 88 143 143 143 F F F 23 460-479 88 88 88 143 143 143 F F F 24 480-499 88 88 88 143 143 143 F F F 25 500-519 90 90 88 144 144 143 F F F <--- 26 520-539 88 88 88 143 143 143 T T F <--- 27 540-559 88 88 88 143 143 143 T T F <--- 28 560-579 90 90 88 144 144 143 T T T <--- 29 580-599 88 88 88 143 143 143 F F F Max gen: 90, Max flush: 145 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 45 Verifying data now No change after verify Loop: 45 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 90 90 90 145 145 145 F F F 1 020-039 88 88 88 143 143 143 F F F 2 040-059 90 90 90 144 144 144 F F F 3 060-079 90 90 90 145 145 145 F F F 4 080-099 82 82 82 133 133 133 F F F 5 100-119 90 90 90 145 145 145 F F F 6 120-139 88 88 88 143 143 143 F F F 7 140-159 84 84 84 137 137 137 F F F 8 160-179 87 87 87 142 142 142 F F F 9 180-199 90 90 90 144 144 144 F F F 10 200-219 90 90 90 145 145 145 F F F 11 220-239 90 90 90 145 145 145 F F F 12 240-259 82 82 82 133 133 133 F F F 13 260-279 88 88 88 143 143 143 F F F 14 280-299 90 90 90 145 145 145 F F F 15 300-319 90 90 90 145 145 145 F F F 16 320-339 82 82 82 133 133 133 F F F 17 340-359 84 84 84 135 135 135 F F F 18 360-379 88 88 88 143 143 143 F F F 19 380-399 78 78 78 126 126 126 F F F 20 400-419 90 90 90 145 145 145 F F F 21 420-439 90 90 90 145 145 145 F F F 22 440-459 88 88 88 143 143 143 F F F 23 460-479 88 88 88 143 143 143 F F F 24 480-499 88 88 88 143 143 143 F F F 25 500-519 90 90 90 144 144 144 F F F 26 520-539 91 91 91 146 146 146 F F F 27 540-559 91 91 91 146 146 146 F F F 28 560-579 91 91 91 146 146 146 F F F 29 580-599 88 88 88 143 143 143 F F F Max gen: 91, Max flush: 146 Begin loop 46 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:17.614031506Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1501,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:17.622000426Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1501,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:17.622279338Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1501} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:17.622305597Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1501} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:17.622314187Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1501} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:17.622325717Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1501} {"msg":"Crucible d66528fb-2539-4883-9358-a49d93c6d89e has session id: e01c498e-6981-4334-88bc-5afe8f20fb2a","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:17.622693596Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1501,"session_id":"e01c498e-6981-4334-88bc-5afe8f20fb2a"} {"msg":"Upstairs opts: Upstairs UUID: d66528fb-2539-4883-9358-a49d93c6d89e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:17.622713926Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1501,"session_id":"e01c498e-6981-4334-88bc-5afe8f20fb2a"} {"msg":"Crucible stats registered with UUID: d66528fb-2539-4883-9358-a49d93c6d89e","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:17.622723975Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1501,"session_id":"e01c498e-6981-4334-88bc-5afe8f20fb2a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:45:17.622733025Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1501,"session_id":"e01c498e-6981-4334-88bc-5afe8f20fb2a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:17.622876491Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1501,"":"io task","client":"1","":"downstairs","session_id":"e01c498e-6981-4334-88bc-5afe8f20fb2a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:17.62289425Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1501,"":"io task","client":"0","":"downstairs","session_id":"e01c498e-6981-4334-88bc-5afe8f20fb2a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:17.62290451Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1501,"":"io task","client":"2","":"downstairs","session_id":"e01c498e-6981-4334-88bc-5afe8f20fb2a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:17.623094455Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1501,"task":"crutest"} {"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:17.623111154Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1501} {"msg":"d66528fb-2539-4883-9358-a49d93c6d89e active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:17.623146273Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1501,"session_id":"e01c498e-6981-4334-88bc-5afe8f20fb2a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:17.623155483Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1501,"client":"0","":"downstairs","session_id":"e01c498e-6981-4334-88bc-5afe8f20fb2a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:17.623167043Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1501,"client":"1","":"downstairs","session_id":"e01c498e-6981-4334-88bc-5afe8f20fb2a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:17.623176632Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1501,"client":"2","":"downstairs","session_id":"e01c498e-6981-4334-88bc-5afe8f20fb2a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:17.623186132Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1501,"":"io task","client":"2","":"downstairs","session_id":"e01c498e-6981-4334-88bc-5afe8f20fb2a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:17.623195952Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1501,"":"io task","client":"1","":"downstairs","session_id":"e01c498e-6981-4334-88bc-5afe8f20fb2a"} {"msg":"ds_connection connected from Ok(127.0.0.1:49173)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:17.62324871Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1501,"":"io task","client":"2","":"downstairs","session_id":"e01c498e-6981-4334-88bc-5afe8f20fb2a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:17.623273809Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1501,"":"io task","client":"0","":"downstairs","session_id":"e01c498e-6981-4334-88bc-5afe8f20fb2a"} {"msg":"ds_connection connected from Ok(127.0.0.1:58888)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:17.623285139Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1501,"":"io task","client":"1","":"downstairs","session_id":"e01c498e-6981-4334-88bc-5afe8f20fb2a"} {"msg":"ds_connection connected from Ok(127.0.0.1:58238)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:17.623304499Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1501,"":"io task","client":"0","":"downstairs","session_id":"e01c498e-6981-4334-88bc-5afe8f20fb2a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:17.623923681Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1501,"client":"0","":"downstairs","session_id":"e01c498e-6981-4334-88bc-5afe8f20fb2a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:17.623966729Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1501,"client":"2","":"downstairs","session_id":"e01c498e-6981-4334-88bc-5afe8f20fb2a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:17.624040687Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1501,"client":"1","":"downstairs","session_id":"e01c498e-6981-4334-88bc-5afe8f20fb2a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:17.62429768Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1501,"session_id":"e01c498e-6981-4334-88bc-5afe8f20fb2a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:17.624333239Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1501,"session_id":"e01c498e-6981-4334-88bc-5afe8f20fb2a"} {"msg":"[0]R flush_numbers[0..12]: [145, 143, 144, 145, 133, 145, 143, 137, 142, 144, 145, 145]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:17.624383857Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1501,"":"downstairs","":"downstairs","session_id":"e01c498e-6981-4334-88bc-5afe8f20fb2a"} {"msg":"[0]R generation[0..12]: [90, 88, 90, 90, 82, 90, 88, 84, 87, 90, 90, 90]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:17.624395497Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1501,"":"downstairs","":"downstairs","session_id":"e01c498e-6981-4334-88bc-5afe8f20fb2a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:17.624409267Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1501,"":"downstairs","":"downstairs","session_id":"e01c498e-6981-4334-88bc-5afe8f20fb2a"} {"msg":"[1]R flush_numbers[0..12]: [145, 143, 144, 145, 133, 145, 143, 137, 142, 144, 145, 145]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:17.624419946Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1501,"":"downstairs","":"downstairs","session_id":"e01c498e-6981-4334-88bc-5afe8f20fb2a"} {"msg":"[1]R generation[0..12]: [90, 88, 90, 90, 82, 90, 88, 84, 87, 90, 90, 90]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:17.624430236Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1501,"":"downstairs","":"downstairs","session_id":"e01c498e-6981-4334-88bc-5afe8f20fb2a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:17.624441076Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1501,"":"downstairs","":"downstairs","session_id":"e01c498e-6981-4334-88bc-5afe8f20fb2a"} {"msg":"[2]R flush_numbers[0..12]: [145, 143, 144, 145, 133, 145, 143, 137, 142, 144, 145, 145]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:17.624451706Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1501,"":"downstairs","":"downstairs","session_id":"e01c498e-6981-4334-88bc-5afe8f20fb2a"} {"msg":"[2]R generation[0..12]: [90, 88, 90, 90, 82, 90, 88, 84, 87, 90, 90, 90]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:17.624462005Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1501,"":"downstairs","":"downstairs","session_id":"e01c498e-6981-4334-88bc-5afe8f20fb2a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:17.624472855Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1501,"":"downstairs","":"downstairs","session_id":"e01c498e-6981-4334-88bc-5afe8f20fb2a"} {"msg":"Max found gen is 92","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:17.624482175Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1501,"":"downstairs","":"downstairs","session_id":"e01c498e-6981-4334-88bc-5afe8f20fb2a"} {"msg":"Generation requested: 92 >= found:92","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:17.624491824Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1501,"":"downstairs","":"downstairs","session_id":"e01c498e-6981-4334-88bc-5afe8f20fb2a"} {"msg":"Next flush: 147","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:17.624501264Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1501,"":"downstairs","":"downstairs","session_id":"e01c498e-6981-4334-88bc-5afe8f20fb2a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:17.624510684Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1501,"":"downstairs","":"downstairs","session_id":"e01c498e-6981-4334-88bc-5afe8f20fb2a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:17.624520634Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1501,"session_id":"e01c498e-6981-4334-88bc-5afe8f20fb2a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:17.624529373Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1501,"client":"0","":"downstairs","session_id":"e01c498e-6981-4334-88bc-5afe8f20fb2a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:17.624539423Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1501,"client":"1","":"downstairs","session_id":"e01c498e-6981-4334-88bc-5afe8f20fb2a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:17.624549313Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1501,"client":"2","":"downstairs","session_id":"e01c498e-6981-4334-88bc-5afe8f20fb2a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:17.624560612Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1501,"session_id":"e01c498e-6981-4334-88bc-5afe8f20fb2a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:17.624569702Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1501,"session_id":"e01c498e-6981-4334-88bc-5afe8f20fb2a"} {"msg":"d66528fb-2539-4883-9358-a49d93c6d89e is now active with session: e01c498e-6981-4334-88bc-5afe8f20fb2a","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:17.624579472Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1501,"session_id":"e01c498e-6981-4334-88bc-5afe8f20fb2a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:17.624588172Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1501,"session_id":"e01c498e-6981-4334-88bc-5afe8f20fb2a"} {"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:17.624597141Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1501} {"msg":"Activated sub_volume d66528fb-2539-4883-9358-a49d93c6d89e","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:17.624605041Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1501} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:17.624613061Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1501,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:17.624935742Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1501,"task":"crutest"} 01/30 Write block 209 len 40960 data: 11 11 9 10 11 10 8 9 10 9 02/30 Read block 529 len 4096 03/30 Read block 341 len 4096 04/30 Flush 05/30 Write block 572 len 24576 data: 8 8 9 8 10 9 06/30 Read block 572 len 40960 07/30 Write block 8 len 8192 data: 11 9 08/30 Read block 129 len 20480 09/30 Read block 364 len 40960 10/30 Flush 11/30 Write block 392 len 20480 data: 6 6 4 5 6 12/30 Write block 243 len 36864 data: 6 7 7 9 11 10 8 9 6 13/30 Write block 36 len 4096 data: 6 14/30 Read block 323 len 20480 15/30 Write block 46 len 4096 data: 5 16/30 Read block 573 len 40960 17/30 Write block 543 len 36864 data: 9 8 8 7 7 9 9 10 6 18/30 Read block 369 len 12288 19/30 Read block 523 len 32768 20/30 Write block 486 len 24576 data: 8 9 9 9 10 11 21/30 Write block 338 len 8192 data: 6 5 22/30 Write block 195 len 24576 data: 7 8 8 9 8 6 23/30 Write block 391 len 8192 data: 6 7 24/30 Read block 354 len 12288 25/30 Write block 353 len 20480 data: 6 6 5 6 6 26/30 Flush 27/30 Write block 323 len 36864 data: 8 10 9 10 10 8 9 9 10 28/30 Write block 327 len 24576 data: 11 9 10 10 11 9 29/30 Read block 145 len 36864 30/30 Write block 407 len 24576 data: 6 6 8 10 10 10 ---------------------------------------------------------------- Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Read 1 Done Done Done false 1002 Acked Read 1 Done Sent Done false 1003 Acked Flush 0 Done Sent Done false 1004 Acked Write 6 Done Sent Done false 1005 Acked Read 10 Done Sent Done false 1006 Acked Write 2 Done Sent Done false 1007 Acked Read 5 Done Sent Done false 1008 Acked Read 10 Done Sent Done false 1009 Acked Flush 0 Done Sent Done false 1010 Acked Write 5 Done Sent Done false 1011 Acked Write 9 Done Sent Done false 1012 Acked Write 1 Done Sent Done false 1013 Acked Read 5 Done Sent Done false 1014 Acked Write 1 Done Sent Done false 1015 Acked Read 10 Done Sent Done false 1016 Acked Write 9 Done Sent Done false 1017 Acked Read 3 Done Sent Done false 1018 Acked Read 8 Done Sent Done false 1019 Acked Write 6 Done Sent Done false 1020 Acked Write 2 Done Sent Done false 1021 Acked Write 6 Done Sent Done false 1022 Acked Write 2 Done Sent Done false 1023 Acked Read 3 Done Sent Done false 1024 Acked Write 5 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 9 Done Sent Done false 1027 Acked Write 6 Done Sent Done false 1028 Acked Read 9 Done Sent Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 28 1 30 Done 29 2 29 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) None Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:45:17.634318641Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1501} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:17.634504656Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1501,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 90 92 148 145 148 F F F <--- 1 020-039 92 88 92 149 143 149 F F F <--- 2 040-059 92 90 92 149 144 149 F F F <--- 3 060-079 90 90 90 145 145 145 F F F 4 080-099 82 82 82 133 133 133 F F F 5 100-119 90 90 90 145 145 145 F F F 6 120-139 88 88 88 143 143 143 F F F 7 140-159 84 84 84 137 137 137 F F F 8 160-179 87 87 87 142 142 142 F F F 9 180-199 92 90 92 149 144 149 F F F <--- 10 200-219 92 90 92 149 145 149 F T F <--- 11 220-239 90 90 90 145 145 145 F F F 12 240-259 92 82 92 149 133 149 F F F <--- 13 260-279 88 88 88 143 143 143 F F F 14 280-299 90 90 90 145 145 145 F F F 15 300-319 90 90 90 145 145 145 F F F 16 320-339 92 82 92 149 133 149 T F T <--- 17 340-359 92 84 92 149 135 149 F F F <--- 18 360-379 88 88 88 143 143 143 F F F 19 380-399 92 78 92 149 126 149 F F F <--- 20 400-419 90 90 90 145 145 145 T F T <--- 21 420-439 90 90 90 145 145 145 F F F 22 440-459 88 88 88 143 143 143 F F F 23 460-479 88 88 88 143 143 143 F F F 24 480-499 92 88 92 149 143 149 F F F <--- 25 500-519 90 90 90 144 144 144 F F F 26 520-539 91 91 91 146 146 146 F F F 27 540-559 92 91 92 149 146 149 F F F <--- 28 560-579 92 91 92 148 146 148 F F F <--- 29 580-599 88 88 88 143 143 143 F F F Max gen: 92, Max flush: 149 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 46 Verifying data now No change after verify Loop: 46 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 148 148 148 F F F 1 020-039 92 92 92 149 149 149 F F F 2 040-059 92 92 92 149 149 149 F F F 3 060-079 90 90 90 145 145 145 F F F 4 080-099 82 82 82 133 133 133 F F F 5 100-119 90 90 90 145 145 145 F F F 6 120-139 88 88 88 143 143 143 F F F 7 140-159 84 84 84 137 137 137 F F F 8 160-179 87 87 87 142 142 142 F F F 9 180-199 92 92 92 149 149 149 F F F 10 200-219 92 92 92 149 149 149 F F F 11 220-239 90 90 90 145 145 145 F F F 12 240-259 92 92 92 149 149 149 F F F 13 260-279 88 88 88 143 143 143 F F F 14 280-299 90 90 90 145 145 145 F F F 15 300-319 90 90 90 145 145 145 F F F 16 320-339 93 93 93 150 150 150 F F F 17 340-359 92 92 92 149 149 149 F F F 18 360-379 88 88 88 143 143 143 F F F 19 380-399 92 92 92 149 149 149 F F F 20 400-419 93 93 93 150 150 150 F F F 21 420-439 90 90 90 145 145 145 F F F 22 440-459 88 88 88 143 143 143 F F F 23 460-479 88 88 88 143 143 143 F F F 24 480-499 92 92 92 149 149 149 F F F 25 500-519 90 90 90 144 144 144 F F F 26 520-539 91 91 91 146 146 146 F F F 27 540-559 92 92 92 149 149 149 F F F 28 560-579 92 92 92 148 148 148 F F F 29 580-599 88 88 88 143 143 143 F F F Max gen: 93, Max flush: 150 Begin loop 47 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:21.877369011Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1509,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:21.885315522Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1509,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:21.885606474Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1509} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:21.885637383Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1509} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:21.885646192Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1509} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:21.885658022Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1509} {"msg":"Crucible 0460d07a-473a-4dde-8b4e-81a6be9172e1 has session id: 3c45f888-9ada-47d7-8c66-b59d49276682","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:21.886020622Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1509,"session_id":"3c45f888-9ada-47d7-8c66-b59d49276682"} {"msg":"Upstairs opts: Upstairs UUID: 0460d07a-473a-4dde-8b4e-81a6be9172e1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:21.886052651Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1509,"session_id":"3c45f888-9ada-47d7-8c66-b59d49276682"} {"msg":"Crucible stats registered with UUID: 0460d07a-473a-4dde-8b4e-81a6be9172e1","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:21.88606371Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1509,"session_id":"3c45f888-9ada-47d7-8c66-b59d49276682"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:45:21.88607316Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1509,"session_id":"3c45f888-9ada-47d7-8c66-b59d49276682"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:21.886200646Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1509,"":"io task","client":"2","":"downstairs","session_id":"3c45f888-9ada-47d7-8c66-b59d49276682"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:21.886233636Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1509,"":"io task","client":"0","":"downstairs","session_id":"3c45f888-9ada-47d7-8c66-b59d49276682"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:21.886244385Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1509,"":"io task","client":"1","":"downstairs","session_id":"3c45f888-9ada-47d7-8c66-b59d49276682"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:21.886348822Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1509,"task":"crutest"} {"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:21.886364532Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1509} {"msg":"0460d07a-473a-4dde-8b4e-81a6be9172e1 active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:21.886400821Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1509,"session_id":"3c45f888-9ada-47d7-8c66-b59d49276682"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:21.88641033Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1509,"client":"0","":"downstairs","session_id":"3c45f888-9ada-47d7-8c66-b59d49276682"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:21.88642023Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1509,"client":"1","":"downstairs","session_id":"3c45f888-9ada-47d7-8c66-b59d49276682"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:21.88642987Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1509,"client":"2","":"downstairs","session_id":"3c45f888-9ada-47d7-8c66-b59d49276682"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:21.88643934Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1509,"":"io task","client":"2","":"downstairs","session_id":"3c45f888-9ada-47d7-8c66-b59d49276682"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:21.886449349Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1509,"":"io task","client":"1","":"downstairs","session_id":"3c45f888-9ada-47d7-8c66-b59d49276682"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:21.886489378Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1509,"":"io task","client":"0","":"downstairs","session_id":"3c45f888-9ada-47d7-8c66-b59d49276682"} {"msg":"ds_connection connected from Ok(127.0.0.1:54942)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:21.886501958Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1509,"":"io task","client":"2","":"downstairs","session_id":"3c45f888-9ada-47d7-8c66-b59d49276682"} {"msg":"ds_connection connected from Ok(127.0.0.1:51466)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:21.886516117Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1509,"":"io task","client":"1","":"downstairs","session_id":"3c45f888-9ada-47d7-8c66-b59d49276682"} {"msg":"ds_connection connected from Ok(127.0.0.1:43910)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:21.886589595Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1509,"":"io task","client":"0","":"downstairs","session_id":"3c45f888-9ada-47d7-8c66-b59d49276682"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:21.887215947Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1509,"client":"2","":"downstairs","session_id":"3c45f888-9ada-47d7-8c66-b59d49276682"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:21.887240836Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1509,"client":"0","":"downstairs","session_id":"3c45f888-9ada-47d7-8c66-b59d49276682"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:21.887314364Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1509,"client":"1","":"downstairs","session_id":"3c45f888-9ada-47d7-8c66-b59d49276682"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:21.887530658Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1509,"session_id":"3c45f888-9ada-47d7-8c66-b59d49276682"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:21.887540008Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1509,"session_id":"3c45f888-9ada-47d7-8c66-b59d49276682"} {"msg":"[0]R flush_numbers[0..12]: [148, 149, 149, 145, 133, 145, 143, 137, 142, 149, 149, 145]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:21.887615006Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1509,"":"downstairs","":"downstairs","session_id":"3c45f888-9ada-47d7-8c66-b59d49276682"} {"msg":"[0]R generation[0..12]: [92, 92, 92, 90, 82, 90, 88, 84, 87, 92, 92, 90]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:21.887625815Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1509,"":"downstairs","":"downstairs","session_id":"3c45f888-9ada-47d7-8c66-b59d49276682"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:21.887636745Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1509,"":"downstairs","":"downstairs","session_id":"3c45f888-9ada-47d7-8c66-b59d49276682"} {"msg":"[1]R flush_numbers[0..12]: [148, 149, 149, 145, 133, 145, 143, 137, 142, 149, 149, 145]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:21.887647365Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1509,"":"downstairs","":"downstairs","session_id":"3c45f888-9ada-47d7-8c66-b59d49276682"} {"msg":"[1]R generation[0..12]: [92, 92, 92, 90, 82, 90, 88, 84, 87, 92, 92, 90]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:21.887657574Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1509,"":"downstairs","":"downstairs","session_id":"3c45f888-9ada-47d7-8c66-b59d49276682"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:21.887668254Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1509,"":"downstairs","":"downstairs","session_id":"3c45f888-9ada-47d7-8c66-b59d49276682"} {"msg":"[2]R flush_numbers[0..12]: [148, 149, 149, 145, 133, 145, 143, 137, 142, 149, 149, 145]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:21.887678684Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1509,"":"downstairs","":"downstairs","session_id":"3c45f888-9ada-47d7-8c66-b59d49276682"} {"msg":"[2]R generation[0..12]: [92, 92, 92, 90, 82, 90, 88, 84, 87, 92, 92, 90]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:21.887691603Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1509,"":"downstairs","":"downstairs","session_id":"3c45f888-9ada-47d7-8c66-b59d49276682"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:21.887702383Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1509,"":"downstairs","":"downstairs","session_id":"3c45f888-9ada-47d7-8c66-b59d49276682"} {"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:21.887711633Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1509,"":"downstairs","":"downstairs","session_id":"3c45f888-9ada-47d7-8c66-b59d49276682"} {"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:21.887721303Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1509,"":"downstairs","":"downstairs","session_id":"3c45f888-9ada-47d7-8c66-b59d49276682"} {"msg":"Next flush: 151","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:21.887730472Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1509,"":"downstairs","":"downstairs","session_id":"3c45f888-9ada-47d7-8c66-b59d49276682"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:21.887739782Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1509,"":"downstairs","":"downstairs","session_id":"3c45f888-9ada-47d7-8c66-b59d49276682"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:21.887749412Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1509,"session_id":"3c45f888-9ada-47d7-8c66-b59d49276682"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:21.887758112Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1509,"client":"0","":"downstairs","session_id":"3c45f888-9ada-47d7-8c66-b59d49276682"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:21.887767731Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1509,"client":"1","":"downstairs","session_id":"3c45f888-9ada-47d7-8c66-b59d49276682"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:21.887777221Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1509,"client":"2","":"downstairs","session_id":"3c45f888-9ada-47d7-8c66-b59d49276682"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:21.887786901Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1509,"session_id":"3c45f888-9ada-47d7-8c66-b59d49276682"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:21.88779585Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1509,"session_id":"3c45f888-9ada-47d7-8c66-b59d49276682"} {"msg":"0460d07a-473a-4dde-8b4e-81a6be9172e1 is now active with session: 3c45f888-9ada-47d7-8c66-b59d49276682","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:21.88780598Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1509,"session_id":"3c45f888-9ada-47d7-8c66-b59d49276682"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:21.88781465Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1509,"session_id":"3c45f888-9ada-47d7-8c66-b59d49276682"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:21.88782854Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1509} {"msg":"Activated sub_volume 0460d07a-473a-4dde-8b4e-81a6be9172e1","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:21.887837859Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1509} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:21.887846109Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1509,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:21.888091372Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1509,"task":"crutest"} 01/30 Write block 511 len 8192 data: 13 9 02/30 Read block 346 len 4096 03/30 Flush 04/30 Flush 05/30 Write block 425 len 20480 data: 7 7 7 6 8 06/30 Write block 85 len 24576 data: 7 6 5 3 4 5 07/30 Flush 08/30 Flush 09/30 Write block 227 len 8192 data: 13 10 10/30 Read block 321 len 32768 11/30 Write block 388 len 40960 data: 7 6 6 7 8 7 5 6 7 7 12/30 Flush 13/30 Write block 532 len 28672 data: 12 11 11 13 11 13 15 14/30 Write block 314 len 8192 data: 7 6 15/30 Read block 590 len 32768 16/30 Read block 563 len 12288 17/30 Write block 237 len 28672 data: 5 4 6 7 8 7 7 18/30 Write block 70 len 4096 data: 7 19/30 Read block 139 len 16384 20/30 Read block 307 len 8192 21/30 Read block 134 len 8192 22/30 Write block 481 len 4096 data: 11 23/30 Read block 575 len 40960 24/30 Read block 252 len 36864 25/30 Read block 97 len 40960 26/30 Write block 499 len 36864 data: 7 8 7 7 8 8 7 7 9 27/30 Read block 441 len 16384 28/30 Write block 154 len 24576 data: 4 3 4 4 6 9 29/30 Read block 134 len 36864 30/30 Read block 316 len 20480 ---------------------------------------------------------------- Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1003 Acked Flush 0 Done Sent Done false 1004 Acked Write 5 Done Sent Done false 1005 Acked Write 6 Done Sent Done false 1006 Acked Flush 0 Done Sent Done false 1007 Acked Flush 0 Done Sent Done false 1008 Acked Write 2 Done Sent Done false 1009 Acked Read 8 Done Sent Done false 1010 Acked Write 10 Done Sent Done false 1011 Acked Flush 0 Done Sent Done false 1012 Acked Write 7 Done Sent Done false 1013 Acked Write 2 Done Sent Done false 1014 Acked Read 8 Done Sent Done false 1015 Acked Read 3 Done Sent Done false 1016 Acked Write 7 Done Sent Done false 1017 Acked Write 1 Done Sent Done false 1018 Acked Read 4 Done Sent Done false 1019 Acked Read 2 Done Sent Done false 1020 Acked Read 2 Done Sent Done false 1021 Acked Write 1 Done Sent Done false 1022 Acked Read 10 Done Sent Done false 1023 Acked Read 9 Done Sent Done false 1024 Acked Read 10 Done Sent Done false 1025 Acked Write 9 Done Sent Done false 1026 Acked Read 4 Done Sent Done false 1027 Acked Write 6 Done Sent Done false 1028 Acked Read 9 Done Sent Done false 1029 Acked Read 5 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 27 0 27 Done 27 0 27 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1011)) Some(JobId(1002)) Some(JobId(1011)) Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:45:21.896791931Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1509} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:21.896974626Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1509,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 148 148 148 F F F 1 020-039 92 92 92 149 149 149 F F F 2 040-059 92 92 92 149 149 149 F F F 3 060-079 90 90 90 145 145 145 T F T <--- 4 080-099 94 82 94 153 133 153 F F F <--- 5 100-119 90 90 90 145 145 145 F F F 6 120-139 88 88 88 143 143 143 F F F 7 140-159 84 84 84 137 137 137 T F T <--- 8 160-179 87 87 87 142 142 142 F F F 9 180-199 92 92 92 149 149 149 F F F 10 200-219 92 92 92 149 149 149 F F F 11 220-239 94 90 94 155 145 155 T F T <--- 12 240-259 92 92 92 149 149 149 T F T <--- 13 260-279 88 88 88 143 143 143 F F F 14 280-299 90 90 90 145 145 145 F F F 15 300-319 90 90 90 145 145 145 T F T <--- 16 320-339 93 93 93 150 150 150 F F F 17 340-359 92 92 92 149 149 149 F F F 18 360-379 88 88 88 143 143 143 F F F 19 380-399 94 92 94 155 149 155 F F F <--- 20 400-419 93 93 93 150 150 150 F F F 21 420-439 94 90 94 153 145 153 F F F <--- 22 440-459 88 88 88 143 143 143 F F F 23 460-479 88 88 88 143 143 143 F F F 24 480-499 92 92 92 149 149 149 T F T <--- 25 500-519 94 94 94 151 151 151 T F T <--- 26 520-539 91 91 91 146 146 146 T F T <--- 27 540-559 92 92 92 149 149 149 F F F 28 560-579 92 92 92 148 148 148 F F F 29 580-599 88 88 88 143 143 143 F F F Max gen: 94, Max flush: 155 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 47 Verifying data now No change after verify Loop: 47 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 148 148 148 F F F 1 020-039 92 92 92 149 149 149 F F F 2 040-059 92 92 92 149 149 149 F F F 3 060-079 95 95 95 156 156 156 F F F 4 080-099 94 94 94 153 153 153 F F F 5 100-119 90 90 90 145 145 145 F F F 6 120-139 88 88 88 143 143 143 F F F 7 140-159 95 95 95 156 156 156 F F F 8 160-179 87 87 87 142 142 142 F F F 9 180-199 92 92 92 149 149 149 F F F 10 200-219 92 92 92 149 149 149 F F F 11 220-239 95 95 95 156 156 156 F F F 12 240-259 95 95 95 156 156 156 F F F 13 260-279 88 88 88 143 143 143 F F F 14 280-299 90 90 90 145 145 145 F F F 15 300-319 95 95 95 156 156 156 F F F 16 320-339 93 93 93 150 150 150 F F F 17 340-359 92 92 92 149 149 149 F F F 18 360-379 88 88 88 143 143 143 F F F 19 380-399 94 94 94 155 155 155 F F F 20 400-419 93 93 93 150 150 150 F F F 21 420-439 94 94 94 153 153 153 F F F 22 440-459 88 88 88 143 143 143 F F F 23 460-479 88 88 88 143 143 143 F F F 24 480-499 95 95 95 156 156 156 F F F 25 500-519 95 95 95 156 156 156 F F F 26 520-539 95 95 95 156 156 156 F F F 27 540-559 92 92 92 149 149 149 F F F 28 560-579 92 92 92 148 148 148 F F F 29 580-599 88 88 88 143 143 143 F F F Max gen: 95, Max flush: 156 Begin loop 48 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:26.141390487Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1517,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:26.149514063Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1517,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:26.149824164Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1517} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:26.149848423Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1517} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:26.149857043Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1517} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:26.149868583Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1517} {"msg":"Crucible 2ec83357-7127-4794-bda4-ba0b1a710216 has session id: 215b31d5-8a01-4fde-be1d-e9141a20cd34","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:26.150175044Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1517,"session_id":"215b31d5-8a01-4fde-be1d-e9141a20cd34"} {"msg":"Upstairs opts: Upstairs UUID: 2ec83357-7127-4794-bda4-ba0b1a710216, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:26.150203723Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1517,"session_id":"215b31d5-8a01-4fde-be1d-e9141a20cd34"} {"msg":"Crucible stats registered with UUID: 2ec83357-7127-4794-bda4-ba0b1a710216","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:26.150214503Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1517,"session_id":"215b31d5-8a01-4fde-be1d-e9141a20cd34"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:45:26.150223882Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1517,"session_id":"215b31d5-8a01-4fde-be1d-e9141a20cd34"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:26.150357058Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1517,"":"io task","client":"2","":"downstairs","session_id":"215b31d5-8a01-4fde-be1d-e9141a20cd34"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:26.150388358Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1517,"":"io task","client":"0","":"downstairs","session_id":"215b31d5-8a01-4fde-be1d-e9141a20cd34"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:26.150399287Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1517,"":"io task","client":"1","":"downstairs","session_id":"215b31d5-8a01-4fde-be1d-e9141a20cd34"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:26.150531763Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1517,"task":"crutest"} {"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:26.150542423Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1517} {"msg":"2ec83357-7127-4794-bda4-ba0b1a710216 active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:26.150558453Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1517,"session_id":"215b31d5-8a01-4fde-be1d-e9141a20cd34"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:26.150567402Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1517,"client":"0","":"downstairs","session_id":"215b31d5-8a01-4fde-be1d-e9141a20cd34"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:26.150576992Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1517,"client":"1","":"downstairs","session_id":"215b31d5-8a01-4fde-be1d-e9141a20cd34"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:26.150586432Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1517,"client":"2","":"downstairs","session_id":"215b31d5-8a01-4fde-be1d-e9141a20cd34"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:26.150596042Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1517,"":"io task","client":"2","":"downstairs","session_id":"215b31d5-8a01-4fde-be1d-e9141a20cd34"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:26.150605911Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1517,"":"io task","client":"0","":"downstairs","session_id":"215b31d5-8a01-4fde-be1d-e9141a20cd34"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:26.150615781Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1517,"":"io task","client":"1","":"downstairs","session_id":"215b31d5-8a01-4fde-be1d-e9141a20cd34"} {"msg":"ds_connection connected from Ok(127.0.0.1:43337)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:26.150682699Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1517,"":"io task","client":"1","":"downstairs","session_id":"215b31d5-8a01-4fde-be1d-e9141a20cd34"} {"msg":"ds_connection connected from Ok(127.0.0.1:36659)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:26.150708558Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1517,"":"io task","client":"2","":"downstairs","session_id":"215b31d5-8a01-4fde-be1d-e9141a20cd34"} {"msg":"ds_connection connected from Ok(127.0.0.1:48143)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:26.150719598Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1517,"":"io task","client":"0","":"downstairs","session_id":"215b31d5-8a01-4fde-be1d-e9141a20cd34"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:26.15135044Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1517,"client":"0","":"downstairs","session_id":"215b31d5-8a01-4fde-be1d-e9141a20cd34"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:26.151368049Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1517,"client":"2","":"downstairs","session_id":"215b31d5-8a01-4fde-be1d-e9141a20cd34"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:26.151379089Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1517,"client":"1","":"downstairs","session_id":"215b31d5-8a01-4fde-be1d-e9141a20cd34"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:26.151665771Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1517,"session_id":"215b31d5-8a01-4fde-be1d-e9141a20cd34"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:26.15167537Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1517,"session_id":"215b31d5-8a01-4fde-be1d-e9141a20cd34"} {"msg":"[0]R flush_numbers[0..12]: [148, 149, 149, 156, 153, 145, 143, 156, 142, 149, 149, 156]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:26.151716589Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1517,"":"downstairs","":"downstairs","session_id":"215b31d5-8a01-4fde-be1d-e9141a20cd34"} {"msg":"[0]R generation[0..12]: [92, 92, 92, 95, 94, 90, 88, 95, 87, 92, 92, 95]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:26.151727579Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1517,"":"downstairs","":"downstairs","session_id":"215b31d5-8a01-4fde-be1d-e9141a20cd34"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:26.151738409Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1517,"":"downstairs","":"downstairs","session_id":"215b31d5-8a01-4fde-be1d-e9141a20cd34"} {"msg":"[1]R flush_numbers[0..12]: [148, 149, 149, 156, 153, 145, 143, 156, 142, 149, 149, 156]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:26.151748958Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1517,"":"downstairs","":"downstairs","session_id":"215b31d5-8a01-4fde-be1d-e9141a20cd34"} {"msg":"[1]R generation[0..12]: [92, 92, 92, 95, 94, 90, 88, 95, 87, 92, 92, 95]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:26.151759258Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1517,"":"downstairs","":"downstairs","session_id":"215b31d5-8a01-4fde-be1d-e9141a20cd34"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:26.151770068Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1517,"":"downstairs","":"downstairs","session_id":"215b31d5-8a01-4fde-be1d-e9141a20cd34"} {"msg":"[2]R flush_numbers[0..12]: [148, 149, 149, 156, 153, 145, 143, 156, 142, 149, 149, 156]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:26.151780537Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1517,"":"downstairs","":"downstairs","session_id":"215b31d5-8a01-4fde-be1d-e9141a20cd34"} {"msg":"[2]R generation[0..12]: [92, 92, 92, 95, 94, 90, 88, 95, 87, 92, 92, 95]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:26.151790647Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1517,"":"downstairs","":"downstairs","session_id":"215b31d5-8a01-4fde-be1d-e9141a20cd34"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:26.151801477Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1517,"":"downstairs","":"downstairs","session_id":"215b31d5-8a01-4fde-be1d-e9141a20cd34"} {"msg":"Max found gen is 96","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:26.151810657Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1517,"":"downstairs","":"downstairs","session_id":"215b31d5-8a01-4fde-be1d-e9141a20cd34"} {"msg":"Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:26.151820346Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1517,"":"downstairs","":"downstairs","session_id":"215b31d5-8a01-4fde-be1d-e9141a20cd34"} {"msg":"Next flush: 157","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:26.151829776Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1517,"":"downstairs","":"downstairs","session_id":"215b31d5-8a01-4fde-be1d-e9141a20cd34"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:26.151839046Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1517,"":"downstairs","":"downstairs","session_id":"215b31d5-8a01-4fde-be1d-e9141a20cd34"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:26.151850255Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1517,"session_id":"215b31d5-8a01-4fde-be1d-e9141a20cd34"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:26.151858895Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1517,"client":"0","":"downstairs","session_id":"215b31d5-8a01-4fde-be1d-e9141a20cd34"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:26.151868415Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1517,"client":"1","":"downstairs","session_id":"215b31d5-8a01-4fde-be1d-e9141a20cd34"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:26.151877895Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1517,"client":"2","":"downstairs","session_id":"215b31d5-8a01-4fde-be1d-e9141a20cd34"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:26.151887544Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1517,"session_id":"215b31d5-8a01-4fde-be1d-e9141a20cd34"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:26.151896394Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1517,"session_id":"215b31d5-8a01-4fde-be1d-e9141a20cd34"} {"msg":"2ec83357-7127-4794-bda4-ba0b1a710216 is now active with session: 215b31d5-8a01-4fde-be1d-e9141a20cd34","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:26.151906314Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1517,"session_id":"215b31d5-8a01-4fde-be1d-e9141a20cd34"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:26.151914884Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1517,"session_id":"215b31d5-8a01-4fde-be1d-e9141a20cd34"} {"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:26.151923853Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1517} {"msg":"Activated sub_volume 2ec83357-7127-4794-bda4-ba0b1a710216","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:26.151931853Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1517} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:26.151939803Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1517,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:26.152262244Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1517,"task":"crutest"} 01/30 Write block 93 len 28672 data: 6 9 9 10 8 7 7 02/30 Write block 402 len 4096 data: 7 03/30 Write block 520 len 12288 data: 10 13 13 04/30 Read block 568 len 28672 05/30 Write block 174 len 40960 data: 7 8 7 8 7 7 6 5 7 7 06/30 Read block 363 len 32768 07/30 Flush 08/30 Write block 16 len 8192 data: 8 8 09/30 Write block 301 len 20480 data: 9 6 7 7 7 10/30 Write block 476 len 28672 data: 11 10 10 9 8 12 8 11/30 Read block 60 len 8192 12/30 Write block 461 len 16384 data: 4 4 4 3 13/30 Write block 228 len 16384 data: 11 11 11 12 14/30 Read block 213 len 28672 15/30 Read block 477 len 36864 16/30 Read block 465 len 4096 17/30 Read block 493 len 24576 18/30 Flush 19/30 Write block 423 len 8192 data: 8 8 20/30 Flush 21/30 Write block 141 len 40960 data: 11 10 9 9 8 9 8 7 5 5 22/30 Read block 440 len 24576 23/30 Read block 406 len 36864 24/30 Write block 180 len 20480 data: 7 6 8 8 10 25/30 Write block 593 len 28672 data: 4 5 6 5 4 3 3 26/30 Read block 583 len 32768 27/30 Read block 310 len 12288 28/30 Read block 147 len 12288 29/30 Write block 159 len 20480 data: 10 9 5 5 6 30/30 Read block 288 len 36864 ---------------------------------------------------------------- Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Write 1 Done Done Done false 1002 Acked Write 3 Done Sent Done false 1003 Acked Read 7 Done Sent Done false 1004 Acked Write 10 Done Sent Done false 1005 Acked Read 8 Done Sent Done false 1006 Acked Flush 0 Done Sent Done false 1007 Acked Write 2 Done Sent Done false 1008 Acked Write 5 Done Sent Done false 1009 Acked Write 7 Done Sent Done false 1010 Acked Read 2 Done Sent Done false 1011 Acked Write 4 Done Sent Done false 1012 Acked Write 4 Done Sent Done false 1013 Acked Read 7 Done Sent Done false 1014 Acked Read 9 Done Sent Done false 1015 Acked Read 1 Done Sent Done false 1016 Acked Read 6 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Write 2 Done Sent Done false 1019 Acked Flush 0 Done Sent Done false 1020 Acked Write 10 Done Sent Done false 1021 Acked Read 6 Done Sent Done false 1022 Acked Read 9 Done Sent Done false 1023 Acked Write 5 Done Sent Done false 1024 Acked Write 7 Done Sent Done false 1025 Acked Read 8 Done Sent Done false 1026 Acked Read 3 Done Sent Done false 1027 Acked Read 3 Done Sent Done false 1028 Acked Write 5 Done Sent Done false 1029 Acked Read 9 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 28 0 28 Done 30 2 30 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) None Some(JobId(1019)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:45:26.16105186Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1517} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:26.161226625Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1517,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 96 92 96 158 148 158 F F F <--- 1 020-039 92 92 92 149 149 149 F F F 2 040-059 92 92 92 149 149 149 F F F 3 060-079 95 95 95 156 156 156 F F F 4 080-099 96 94 96 157 153 157 F T F <--- 5 100-119 90 90 90 145 145 145 F F F 6 120-139 88 88 88 143 143 143 F F F 7 140-159 95 95 95 156 156 156 T F T <--- 8 160-179 96 87 96 157 142 157 T F T <--- 9 180-199 96 92 96 157 149 157 T F T <--- 10 200-219 92 92 92 149 149 149 F F F 11 220-239 96 95 96 158 156 158 F F F <--- 12 240-259 95 95 95 156 156 156 F F F 13 260-279 88 88 88 143 143 143 F F F 14 280-299 90 90 90 145 145 145 F F F 15 300-319 96 95 96 158 156 158 F F F <--- 16 320-339 93 93 93 150 150 150 F F F 17 340-359 92 92 92 149 149 149 F F F 18 360-379 88 88 88 143 143 143 F F F 19 380-399 94 94 94 155 155 155 F F F 20 400-419 96 93 96 157 150 157 F T F <--- 21 420-439 96 94 96 159 153 159 F F F <--- 22 440-459 88 88 88 143 143 143 F F F 23 460-479 96 88 96 158 143 158 F F F <--- 24 480-499 96 95 96 158 156 158 F F F <--- 25 500-519 95 95 95 156 156 156 F F F 26 520-539 96 95 96 157 156 157 F F F <--- 27 540-559 92 92 92 149 149 149 F F F 28 560-579 92 92 92 148 148 148 F F F 29 580-599 88 88 88 143 143 143 T F T <--- Max gen: 96, Max flush: 159 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 48 Verifying data now No change after verify Loop: 48 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 96 96 96 158 158 158 F F F 1 020-039 92 92 92 149 149 149 F F F 2 040-059 92 92 92 149 149 149 F F F 3 060-079 95 95 95 156 156 156 F F F 4 080-099 96 96 96 157 157 157 F F F 5 100-119 90 90 90 145 145 145 F F F 6 120-139 88 88 88 143 143 143 F F F 7 140-159 97 97 97 160 160 160 F F F 8 160-179 97 97 97 160 160 160 F F F 9 180-199 97 97 97 160 160 160 F F F 10 200-219 92 92 92 149 149 149 F F F 11 220-239 96 96 96 158 158 158 F F F 12 240-259 95 95 95 156 156 156 F F F 13 260-279 88 88 88 143 143 143 F F F 14 280-299 90 90 90 145 145 145 F F F 15 300-319 96 96 96 158 158 158 F F F 16 320-339 93 93 93 150 150 150 F F F 17 340-359 92 92 92 149 149 149 F F F 18 360-379 88 88 88 143 143 143 F F F 19 380-399 94 94 94 155 155 155 F F F 20 400-419 96 96 96 157 157 157 F F F 21 420-439 96 96 96 159 159 159 F F F 22 440-459 88 88 88 143 143 143 F F F 23 460-479 96 96 96 158 158 158 F F F 24 480-499 96 96 96 158 158 158 F F F 25 500-519 95 95 95 156 156 156 F F F 26 520-539 96 96 96 157 157 157 F F F 27 540-559 92 92 92 149 149 149 F F F 28 560-579 92 92 92 148 148 148 F F F 29 580-599 97 97 97 160 160 160 F F F Max gen: 97, Max flush: 160 Begin loop 49 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:30.410696721Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1525,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:30.418724519Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1525,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:30.41905203Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1525} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:30.419077859Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1525} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:30.419089019Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1525} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:30.419100598Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1525} {"msg":"Crucible 5f765fc2-d0e9-4a1f-9cfd-07677edf9045 has session id: 73bbbd6c-1376-4e81-ba2a-c1f4a06fa8a9","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:30.419452968Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1525,"session_id":"73bbbd6c-1376-4e81-ba2a-c1f4a06fa8a9"} {"msg":"Upstairs opts: Upstairs UUID: 5f765fc2-d0e9-4a1f-9cfd-07677edf9045, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:30.419472927Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1525,"session_id":"73bbbd6c-1376-4e81-ba2a-c1f4a06fa8a9"} {"msg":"Crucible stats registered with UUID: 5f765fc2-d0e9-4a1f-9cfd-07677edf9045","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:30.419482587Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1525,"session_id":"73bbbd6c-1376-4e81-ba2a-c1f4a06fa8a9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:45:30.419491517Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1525,"session_id":"73bbbd6c-1376-4e81-ba2a-c1f4a06fa8a9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:30.419639793Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1525,"":"io task","client":"1","":"downstairs","session_id":"73bbbd6c-1376-4e81-ba2a-c1f4a06fa8a9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:30.419671092Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1525,"":"io task","client":"0","":"downstairs","session_id":"73bbbd6c-1376-4e81-ba2a-c1f4a06fa8a9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:30.419682101Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1525,"":"io task","client":"2","":"downstairs","session_id":"73bbbd6c-1376-4e81-ba2a-c1f4a06fa8a9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:30.419814058Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1525,"task":"crutest"} {"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:30.419832107Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1525} {"msg":"5f765fc2-d0e9-4a1f-9cfd-07677edf9045 active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:30.419863166Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1525,"session_id":"73bbbd6c-1376-4e81-ba2a-c1f4a06fa8a9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:30.419872486Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1525,"client":"0","":"downstairs","session_id":"73bbbd6c-1376-4e81-ba2a-c1f4a06fa8a9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:30.419882256Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1525,"client":"1","":"downstairs","session_id":"73bbbd6c-1376-4e81-ba2a-c1f4a06fa8a9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:30.419891955Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1525,"client":"2","":"downstairs","session_id":"73bbbd6c-1376-4e81-ba2a-c1f4a06fa8a9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:30.419903725Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1525,"":"io task","client":"2","":"downstairs","session_id":"73bbbd6c-1376-4e81-ba2a-c1f4a06fa8a9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:30.419913965Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1525,"":"io task","client":"0","":"downstairs","session_id":"73bbbd6c-1376-4e81-ba2a-c1f4a06fa8a9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:30.419924084Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1525,"":"io task","client":"1","":"downstairs","session_id":"73bbbd6c-1376-4e81-ba2a-c1f4a06fa8a9"} {"msg":"ds_connection connected from Ok(127.0.0.1:46371)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:30.419977673Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1525,"":"io task","client":"1","":"downstairs","session_id":"73bbbd6c-1376-4e81-ba2a-c1f4a06fa8a9"} {"msg":"ds_connection connected from Ok(127.0.0.1:54616)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:30.419993662Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1525,"":"io task","client":"0","":"downstairs","session_id":"73bbbd6c-1376-4e81-ba2a-c1f4a06fa8a9"} {"msg":"ds_connection connected from Ok(127.0.0.1:62253)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:30.420004262Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1525,"":"io task","client":"2","":"downstairs","session_id":"73bbbd6c-1376-4e81-ba2a-c1f4a06fa8a9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:30.420748891Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1525,"client":"2","":"downstairs","session_id":"73bbbd6c-1376-4e81-ba2a-c1f4a06fa8a9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:30.42076717Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1525,"client":"1","":"downstairs","session_id":"73bbbd6c-1376-4e81-ba2a-c1f4a06fa8a9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:30.42077801Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1525,"client":"0","":"downstairs","session_id":"73bbbd6c-1376-4e81-ba2a-c1f4a06fa8a9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:30.421060572Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1525,"session_id":"73bbbd6c-1376-4e81-ba2a-c1f4a06fa8a9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:30.421070431Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1525,"session_id":"73bbbd6c-1376-4e81-ba2a-c1f4a06fa8a9"} {"msg":"[0]R flush_numbers[0..12]: [158, 149, 149, 156, 157, 145, 143, 160, 160, 160, 149, 158]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:30.42110879Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1525,"":"downstairs","":"downstairs","session_id":"73bbbd6c-1376-4e81-ba2a-c1f4a06fa8a9"} {"msg":"[0]R generation[0..12]: [96, 92, 92, 95, 96, 90, 88, 97, 97, 97, 92, 96]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:30.42111985Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1525,"":"downstairs","":"downstairs","session_id":"73bbbd6c-1376-4e81-ba2a-c1f4a06fa8a9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:30.42113078Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1525,"":"downstairs","":"downstairs","session_id":"73bbbd6c-1376-4e81-ba2a-c1f4a06fa8a9"} {"msg":"[1]R flush_numbers[0..12]: [158, 149, 149, 156, 157, 145, 143, 160, 160, 160, 149, 158]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:30.421141529Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1525,"":"downstairs","":"downstairs","session_id":"73bbbd6c-1376-4e81-ba2a-c1f4a06fa8a9"} {"msg":"[1]R generation[0..12]: [96, 92, 92, 95, 96, 90, 88, 97, 97, 97, 92, 96]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:30.421153349Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1525,"":"downstairs","":"downstairs","session_id":"73bbbd6c-1376-4e81-ba2a-c1f4a06fa8a9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:30.421164339Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1525,"":"downstairs","":"downstairs","session_id":"73bbbd6c-1376-4e81-ba2a-c1f4a06fa8a9"} {"msg":"[2]R flush_numbers[0..12]: [158, 149, 149, 156, 157, 145, 143, 160, 160, 160, 149, 158]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:30.421174958Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1525,"":"downstairs","":"downstairs","session_id":"73bbbd6c-1376-4e81-ba2a-c1f4a06fa8a9"} {"msg":"[2]R generation[0..12]: [96, 92, 92, 95, 96, 90, 88, 97, 97, 97, 92, 96]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:30.421185438Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1525,"":"downstairs","":"downstairs","session_id":"73bbbd6c-1376-4e81-ba2a-c1f4a06fa8a9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:30.421196168Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1525,"":"downstairs","":"downstairs","session_id":"73bbbd6c-1376-4e81-ba2a-c1f4a06fa8a9"} {"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:30.421205647Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1525,"":"downstairs","":"downstairs","session_id":"73bbbd6c-1376-4e81-ba2a-c1f4a06fa8a9"} {"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:30.421215307Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1525,"":"downstairs","":"downstairs","session_id":"73bbbd6c-1376-4e81-ba2a-c1f4a06fa8a9"} {"msg":"Next flush: 161","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:30.421224737Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1525,"":"downstairs","":"downstairs","session_id":"73bbbd6c-1376-4e81-ba2a-c1f4a06fa8a9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:30.421234167Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1525,"":"downstairs","":"downstairs","session_id":"73bbbd6c-1376-4e81-ba2a-c1f4a06fa8a9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:30.421244296Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1525,"session_id":"73bbbd6c-1376-4e81-ba2a-c1f4a06fa8a9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:30.421253076Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1525,"client":"0","":"downstairs","session_id":"73bbbd6c-1376-4e81-ba2a-c1f4a06fa8a9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:30.421262946Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1525,"client":"1","":"downstairs","session_id":"73bbbd6c-1376-4e81-ba2a-c1f4a06fa8a9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:30.421272706Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1525,"client":"2","":"downstairs","session_id":"73bbbd6c-1376-4e81-ba2a-c1f4a06fa8a9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:30.421282765Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1525,"session_id":"73bbbd6c-1376-4e81-ba2a-c1f4a06fa8a9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:30.421291945Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1525,"session_id":"73bbbd6c-1376-4e81-ba2a-c1f4a06fa8a9"} {"msg":"5f765fc2-d0e9-4a1f-9cfd-07677edf9045 is now active with session: 73bbbd6c-1376-4e81-ba2a-c1f4a06fa8a9","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:30.421302985Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1525,"session_id":"73bbbd6c-1376-4e81-ba2a-c1f4a06fa8a9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:30.421311764Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1525,"session_id":"73bbbd6c-1376-4e81-ba2a-c1f4a06fa8a9"} {"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:30.421320944Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1525} {"msg":"Activated sub_volume 5f765fc2-d0e9-4a1f-9cfd-07677edf9045","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:30.421328864Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1525} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:30.421341624Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1525,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:30.421645185Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1525,"task":"crutest"} 01/30 Write block 286 len 24576 data: 6 6 6 7 8 8 02/30 Read block 407 len 12288 03/30 Write block 22 len 24576 data: 8 6 5 5 3 3 04/30 Read block 540 len 16384 05/30 Write block 485 len 24576 data: 8 9 10 10 10 11 06/30 Write block 154 len 32768 data: 5 4 5 5 7 11 10 6 07/30 Read block 345 len 8192 08/30 Read block 389 len 40960 09/30 Read block 423 len 40960 10/30 Flush 11/30 Flush 12/30 Write block 395 len 8192 data: 7 8 13/30 Read block 465 len 4096 14/30 Read block 153 len 8192 15/30 Write block 163 len 4096 data: 7 16/30 Read block 302 len 8192 17/30 Write block 111 len 8192 data: 6 8 18/30 Read block 189 len 4096 19/30 Read block 498 len 36864 20/30 Read block 293 len 12288 21/30 Write block 18 len 32768 data: 8 5 5 4 9 7 6 6 22/30 Flush 23/30 Write block 395 len 8192 data: 8 9 24/30 Read block 267 len 8192 25/30 Read block 488 len 16384 26/30 Flush 27/30 Write block 36 len 24576 data: 7 6 5 4 4 5 28/30 Write block 341 len 12288 data: 7 6 7 29/30 Write block 424 len 40960 data: 9 8 8 8 7 9 8 7 6 5 30/30 Write block 151 len 4096 data: 5 ---------------------------------------------------------------- Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Read 3 Done Done Done false 1002 Acked Write 6 Done Sent Done false 1003 Acked Read 4 Done Sent Done false 1004 Acked Write 6 Done Sent Done false 1005 Acked Write 8 Done Sent Done false 1006 Acked Read 2 Done Sent Done false 1007 Acked Read 10 Done Sent Done false 1008 Acked Read 10 Done Sent Done false 1009 Acked Flush 0 Done Sent Done false 1010 Acked Flush 0 Done Sent Done false 1011 Acked Write 2 Done Sent Done false 1012 Acked Read 1 Done Sent Done false 1013 Acked Read 2 Done Sent Done false 1014 Acked Write 1 Done Sent Done false 1015 Acked Read 2 Done Sent Done false 1016 Acked Write 2 Done Sent Done false 1017 Acked Read 1 Done Sent Done false 1018 Acked Read 9 Done Sent Done false 1019 Acked Read 3 Done Sent Done false 1020 Acked Write 8 Done Sent Done false 1021 Acked Flush 0 Done Sent Done false 1022 Acked Write 2 Done Sent Done false 1023 Acked Read 2 Done Sent Done false 1024 Acked Read 4 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 6 Sent Sent Sent false 1027 Acked Write 3 Sent Sent Sent false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 4 28 4 36 Done 26 2 26 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) None Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:45:30.4294502Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1525} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:30.429610595Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1525,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 96 98 163 158 163 F F F <--- 1 020-039 98 92 98 163 149 163 T F T <--- 2 040-059 92 92 92 149 149 149 T F T <--- 3 060-079 95 95 95 156 156 156 F F F 4 080-099 96 96 96 157 157 157 F F F 5 100-119 98 90 98 163 145 163 F F F <--- 6 120-139 88 88 88 143 143 143 F F F 7 140-159 98 97 98 161 160 161 T F T <--- 8 160-179 98 97 98 163 160 163 F F F <--- 9 180-199 97 97 97 160 160 160 F F F 10 200-219 92 92 92 149 149 149 F F F 11 220-239 96 96 96 158 158 158 F F F 12 240-259 95 95 95 156 156 156 F F F 13 260-279 88 88 88 143 143 143 F F F 14 280-299 98 90 98 161 145 161 F T F <--- 15 300-319 96 96 96 158 158 158 F F F 16 320-339 93 93 93 150 150 150 F F F 17 340-359 92 92 92 149 149 149 T F T <--- 18 360-379 88 88 88 143 143 143 F F F 19 380-399 98 94 98 164 155 164 F F F <--- 20 400-419 96 96 96 157 157 157 F F F 21 420-439 96 96 96 159 159 159 T F T <--- 22 440-459 88 88 88 143 143 143 F F F 23 460-479 96 96 96 158 158 158 F F F 24 480-499 98 96 98 161 158 161 F F F <--- 25 500-519 95 95 95 156 156 156 F F F 26 520-539 96 96 96 157 157 157 F F F 27 540-559 92 92 92 149 149 149 F F F 28 560-579 92 92 92 148 148 148 F F F 29 580-599 97 97 97 160 160 160 F F F Max gen: 98, Max flush: 164 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 49 Verifying data now No change after verify Loop: 49 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 163 163 163 F F F 1 020-039 99 99 99 165 165 165 F F F 2 040-059 99 99 99 165 165 165 F F F 3 060-079 95 95 95 156 156 156 F F F 4 080-099 96 96 96 157 157 157 F F F 5 100-119 98 98 98 163 163 163 F F F 6 120-139 88 88 88 143 143 143 F F F 7 140-159 99 99 99 165 165 165 F F F 8 160-179 98 98 98 163 163 163 F F F 9 180-199 97 97 97 160 160 160 F F F 10 200-219 92 92 92 149 149 149 F F F 11 220-239 96 96 96 158 158 158 F F F 12 240-259 95 95 95 156 156 156 F F F 13 260-279 88 88 88 143 143 143 F F F 14 280-299 98 98 98 161 161 161 F F F 15 300-319 96 96 96 158 158 158 F F F 16 320-339 93 93 93 150 150 150 F F F 17 340-359 99 99 99 165 165 165 F F F 18 360-379 88 88 88 143 143 143 F F F 19 380-399 98 98 98 164 164 164 F F F 20 400-419 96 96 96 157 157 157 F F F 21 420-439 99 99 99 165 165 165 F F F 22 440-459 88 88 88 143 143 143 F F F 23 460-479 96 96 96 158 158 158 F F F 24 480-499 98 98 98 161 161 161 F F F 25 500-519 95 95 95 156 156 156 F F F 26 520-539 96 96 96 157 157 157 F F F 27 540-559 92 92 92 149 149 149 F F F 28 560-579 92 92 92 148 148 148 F F F 29 580-599 97 97 97 160 160 160 F F F Max gen: 99, Max flush: 165 Begin loop 50 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:34.674359647Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1533,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:45:34.681443172Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1533,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:34.681498541Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1533,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:34.688295925Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1533,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:34.688560517Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1533} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:34.688585876Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1533} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:34.688594406Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1533} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:34.688605816Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1533} {"msg":"Crucible d10c9fef-c8e3-434c-8d74-f323846bc461 has session id: 236daf66-dbed-4964-bff5-9f28bc7536ec","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:34.688968725Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1533,"session_id":"236daf66-dbed-4964-bff5-9f28bc7536ec"} {"msg":"Upstairs opts: Upstairs UUID: d10c9fef-c8e3-434c-8d74-f323846bc461, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:34.689000185Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1533,"session_id":"236daf66-dbed-4964-bff5-9f28bc7536ec"} {"msg":"Crucible stats registered with UUID: d10c9fef-c8e3-434c-8d74-f323846bc461","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:34.689011424Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1533,"session_id":"236daf66-dbed-4964-bff5-9f28bc7536ec"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:45:34.689020604Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1533,"session_id":"236daf66-dbed-4964-bff5-9f28bc7536ec"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:34.68915377Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1533,"":"io task","client":"1","":"downstairs","session_id":"236daf66-dbed-4964-bff5-9f28bc7536ec"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:34.68917113Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1533,"":"io task","client":"0","":"downstairs","session_id":"236daf66-dbed-4964-bff5-9f28bc7536ec"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:34.689181399Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1533,"":"io task","client":"2","":"downstairs","session_id":"236daf66-dbed-4964-bff5-9f28bc7536ec"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:34.689328535Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1533,"task":"crutest"} {"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:34.689343445Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1533} {"msg":"d10c9fef-c8e3-434c-8d74-f323846bc461 active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:34.689366994Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1533,"session_id":"236daf66-dbed-4964-bff5-9f28bc7536ec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:34.689376304Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1533,"client":"0","":"downstairs","session_id":"236daf66-dbed-4964-bff5-9f28bc7536ec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:34.689386033Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1533,"client":"1","":"downstairs","session_id":"236daf66-dbed-4964-bff5-9f28bc7536ec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:34.689395723Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1533,"client":"2","":"downstairs","session_id":"236daf66-dbed-4964-bff5-9f28bc7536ec"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:34.689405343Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1533,"":"io task","client":"2","":"downstairs","session_id":"236daf66-dbed-4964-bff5-9f28bc7536ec"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:34.689415443Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1533,"":"io task","client":"0","":"downstairs","session_id":"236daf66-dbed-4964-bff5-9f28bc7536ec"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:34.689425332Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1533,"":"io task","client":"1","":"downstairs","session_id":"236daf66-dbed-4964-bff5-9f28bc7536ec"} {"msg":"ds_connection connected from Ok(127.0.0.1:40413)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:34.689463061Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1533,"":"io task","client":"2","":"downstairs","session_id":"236daf66-dbed-4964-bff5-9f28bc7536ec"} {"msg":"ds_connection connected from Ok(127.0.0.1:43037)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:34.689475441Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1533,"":"io task","client":"0","":"downstairs","session_id":"236daf66-dbed-4964-bff5-9f28bc7536ec"} {"msg":"ds_connection connected from Ok(127.0.0.1:63388)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:34.68948575Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1533,"":"io task","client":"1","":"downstairs","session_id":"236daf66-dbed-4964-bff5-9f28bc7536ec"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:34.69018327Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1533,"client":"1","":"downstairs","session_id":"236daf66-dbed-4964-bff5-9f28bc7536ec"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:34.69021204Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1533,"client":"0","":"downstairs","session_id":"236daf66-dbed-4964-bff5-9f28bc7536ec"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:34.690223309Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1533,"client":"2","":"downstairs","session_id":"236daf66-dbed-4964-bff5-9f28bc7536ec"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:34.690485022Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1533,"session_id":"236daf66-dbed-4964-bff5-9f28bc7536ec"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:34.690520731Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1533,"session_id":"236daf66-dbed-4964-bff5-9f28bc7536ec"} {"msg":"[0]R flush_numbers[0..12]: [163, 165, 165, 156, 157, 163, 143, 165, 163, 160, 149, 158]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:34.690568859Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1533,"":"downstairs","":"downstairs","session_id":"236daf66-dbed-4964-bff5-9f28bc7536ec"} {"msg":"[0]R generation[0..12]: [98, 99, 99, 95, 96, 98, 88, 99, 98, 97, 92, 96]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:34.690579739Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1533,"":"downstairs","":"downstairs","session_id":"236daf66-dbed-4964-bff5-9f28bc7536ec"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:34.690590719Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1533,"":"downstairs","":"downstairs","session_id":"236daf66-dbed-4964-bff5-9f28bc7536ec"} {"msg":"[1]R flush_numbers[0..12]: [163, 165, 165, 156, 157, 163, 143, 165, 163, 160, 149, 158]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:34.690601348Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1533,"":"downstairs","":"downstairs","session_id":"236daf66-dbed-4964-bff5-9f28bc7536ec"} {"msg":"[1]R generation[0..12]: [98, 99, 99, 95, 96, 98, 88, 99, 98, 97, 92, 96]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:34.690611628Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1533,"":"downstairs","":"downstairs","session_id":"236daf66-dbed-4964-bff5-9f28bc7536ec"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:34.690622418Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1533,"":"downstairs","":"downstairs","session_id":"236daf66-dbed-4964-bff5-9f28bc7536ec"} {"msg":"[2]R flush_numbers[0..12]: [163, 165, 165, 156, 157, 163, 143, 165, 163, 160, 149, 158]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:34.690633047Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1533,"":"downstairs","":"downstairs","session_id":"236daf66-dbed-4964-bff5-9f28bc7536ec"} {"msg":"[2]R generation[0..12]: [98, 99, 99, 95, 96, 98, 88, 99, 98, 97, 92, 96]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:34.690645757Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1533,"":"downstairs","":"downstairs","session_id":"236daf66-dbed-4964-bff5-9f28bc7536ec"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:34.690656667Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1533,"":"downstairs","":"downstairs","session_id":"236daf66-dbed-4964-bff5-9f28bc7536ec"} {"msg":"Max found gen is 100","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:34.690666086Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1533,"":"downstairs","":"downstairs","session_id":"236daf66-dbed-4964-bff5-9f28bc7536ec"} {"msg":"Generation requested: 100 >= found:100","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:34.690675686Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1533,"":"downstairs","":"downstairs","session_id":"236daf66-dbed-4964-bff5-9f28bc7536ec"} {"msg":"Next flush: 166","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:34.690684886Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1533,"":"downstairs","":"downstairs","session_id":"236daf66-dbed-4964-bff5-9f28bc7536ec"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:34.690694136Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1533,"":"downstairs","":"downstairs","session_id":"236daf66-dbed-4964-bff5-9f28bc7536ec"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:34.690703775Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1533,"session_id":"236daf66-dbed-4964-bff5-9f28bc7536ec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:34.690712375Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1533,"client":"0","":"downstairs","session_id":"236daf66-dbed-4964-bff5-9f28bc7536ec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:34.690722005Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1533,"client":"1","":"downstairs","session_id":"236daf66-dbed-4964-bff5-9f28bc7536ec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:34.690731655Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1533,"client":"2","":"downstairs","session_id":"236daf66-dbed-4964-bff5-9f28bc7536ec"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:34.690741384Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1533,"session_id":"236daf66-dbed-4964-bff5-9f28bc7536ec"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:34.690750534Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1533,"session_id":"236daf66-dbed-4964-bff5-9f28bc7536ec"} {"msg":"d10c9fef-c8e3-434c-8d74-f323846bc461 is now active with session: 236daf66-dbed-4964-bff5-9f28bc7536ec","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:34.690760354Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1533,"session_id":"236daf66-dbed-4964-bff5-9f28bc7536ec"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:34.690769163Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1533,"session_id":"236daf66-dbed-4964-bff5-9f28bc7536ec"} {"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:34.690778353Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1533} {"msg":"Activated sub_volume d10c9fef-c8e3-434c-8d74-f323846bc461","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:34.690786153Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 pid":1533} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:34.690802033Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1533,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:34.691058735Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1533,"task":"crutest"} 01/30 Write block 455 len 28672 data: 6 8 9 8 6 5 5 02/30 Read block 195 len 16384 03/30 Read block 191 len 36864 04/30 Read block 560 len 4096 05/30 Read block 142 len 4096 06/30 Write block 166 len 8192 data: 10 9 07/30 Flush 08/30 Write block 376 len 40960 data: 11 11 12 13 12 12 12 9 9 9 09/30 Read block 60 len 32768 10/30 Write block 353 len 20480 data: 7 7 6 7 7 11/30 Read block 549 len 28672 12/30 Write block 173 len 36864 data: 8 8 9 8 9 8 8 8 7 13/30 Write block 116 len 12288 data: 8 8 7 14/30 Flush 15/30 Write block 511 len 36864 data: 14 10 6 6 6 6 7 9 11 16/30 Read block 331 len 20480 17/30 Read block 13 len 40960 18/30 Read block 354 len 8192 19/30 Write block 34 len 16384 data: 8 7 8 7 20/30 Read block 232 len 36864 21/30 Write block 295 len 24576 data: 10 10 11 8 8 9 22/30 Read block 137 len 24576 23/30 Read block 572 len 12288 24/30 Flush 25/30 Write block 323 len 28672 data: 9 11 10 11 12 10 11 26/30 Read block 586 len 20480 27/30 Read block 337 len 8192 28/30 Write block 387 len 40960 data: 8 8 7 7 8 9 8 6 9 10 29/30 Write block 416 len 4096 data: 7 30/30 Write block 582 len 36864 data: 8 8 8 8 8 11 10 6 6 ---------------------------------------------------------------- Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Sent Done Done false 1001 Acked Read 4 Sent Done Done false 1002 Acked Read 9 Sent Done Done false 1003 Acked Read 1 Sent Done Done false 1004 Acked Read 1 Sent Done Done false 1005 Acked Write 2 Sent Done Done false 1006 Acked Flush 0 Sent Done Done false 1007 Acked Write 10 Sent Done Done false 1008 Acked Read 8 Sent Done Done false 1009 Acked Write 5 Sent Done Done false 1010 Acked Read 7 Sent Done Done false 1011 Acked Write 9 Sent Done Done false 1012 Acked Write 3 Sent Done Done false 1013 Acked Flush 0 Sent Done Done false 1014 Acked Write 9 Sent Done Done false 1015 Acked Read 5 Sent Done Done false 1016 Acked Read 10 Sent Done Done false 1017 Acked Read 2 Sent Done Done false 1018 Acked Write 4 Sent Done Done false 1019 Acked Read 9 Sent Done Done false 1020 Acked Write 6 Sent Done Done false 1021 Acked Read 6 Sent Done Done false 1022 Acked Read 3 Sent Done Done false 1023 Acked Flush 0 Sent Done Done false 1024 Acked Write 7 Sent Done Done false 1025 Acked Read 5 Sent Done Done false 1026 Acked Read 2 Sent Done Done false 1027 Acked Write 10 Sent Sent Sent false 1028 Acked Write 1 Sent Sent Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 3 3 36 Done 0 27 27 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:45:34.700706707Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1533} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:34.70092846Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1533,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 163 163 163 F F F 1 020-039 99 100 100 165 168 168 F F F <--- 2 040-059 99 99 99 165 165 165 F F F 3 060-079 95 95 95 156 156 156 F F F 4 080-099 96 96 96 157 157 157 F F F 5 100-119 98 100 100 163 167 167 F F F <--- 6 120-139 88 88 88 143 143 143 F F F 7 140-159 99 99 99 165 165 165 F F F 8 160-179 98 100 100 163 167 167 F F F <--- 9 180-199 97 100 100 160 167 167 F F F <--- 10 200-219 92 92 92 149 149 149 F F F 11 220-239 96 96 96 158 158 158 F F F 12 240-259 95 95 95 156 156 156 F F F 13 260-279 88 88 88 143 143 143 F F F 14 280-299 98 100 100 161 168 168 F F F <--- 15 300-319 96 100 100 158 168 168 F F F <--- 16 320-339 93 93 93 150 150 150 F T T <--- 17 340-359 99 100 100 165 167 167 F F F <--- 18 360-379 88 100 100 143 167 167 F F F <--- 19 380-399 98 100 100 164 167 167 F T T <--- 20 400-419 96 96 96 157 157 157 F T T <--- 21 420-439 99 99 99 165 165 165 F F F 22 440-459 88 100 100 143 166 166 F F F <--- 23 460-479 96 100 100 158 166 166 F F F <--- 24 480-499 98 98 98 161 161 161 F F F 25 500-519 95 100 100 156 168 168 F F F <--- 26 520-539 96 96 96 157 157 157 F F F 27 540-559 92 92 92 149 149 149 F F F 28 560-579 92 92 92 148 148 148 F F F 29 580-599 97 97 97 160 160 160 F T T <--- Max gen: 100, Max flush: 168 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 50 Verifying data now No change after verify Loop: 50 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 163 163 163 F F F 1 020-039 100 100 100 168 168 168 F F F 2 040-059 99 99 99 165 165 165 F F F 3 060-079 95 95 95 156 156 156 F F F 4 080-099 96 96 96 157 157 157 F F F 5 100-119 100 100 100 167 167 167 F F F 6 120-139 88 88 88 143 143 143 F F F 7 140-159 99 99 99 165 165 165 F F F 8 160-179 100 100 100 167 167 167 F F F 9 180-199 100 100 100 167 167 167 F F F 10 200-219 92 92 92 149 149 149 F F F 11 220-239 96 96 96 158 158 158 F F F 12 240-259 95 95 95 156 156 156 F F F 13 260-279 88 88 88 143 143 143 F F F 14 280-299 100 100 100 168 168 168 F F F 15 300-319 100 100 100 168 168 168 F F F 16 320-339 101 101 101 169 169 169 F F F 17 340-359 100 100 100 167 167 167 F F F 18 360-379 100 100 100 167 167 167 F F F 19 380-399 101 101 101 169 169 169 F F F 20 400-419 101 101 101 169 169 169 F F F 21 420-439 99 99 99 165 165 165 F F F 22 440-459 100 100 100 166 166 166 F F F 23 460-479 100 100 100 166 166 166 F F F 24 480-499 98 98 98 161 161 161 F F F 25 500-519 100 100 100 168 168 168 F F F 26 520-539 96 96 96 157 157 157 F F F 27 540-559 92 92 92 149 149 149 F F F 28 560-579 92 92 92 148 148 148 F F F 29 580-599 101 101 101 169 169 169 F F F Max gen: 101, Max flush: 169 Begin loop 51 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:38.960003569Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1541,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:38.968071066Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1541,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:38.968353458Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1541} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:38.968378597Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1541} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:38.968387537Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1541} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:38.968399166Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1541} {"msg":"Crucible 59470ed2-b8ec-48d2-84c1-3ddc35dd7852 has session id: 1c9476dc-61e8-49a6-9c9a-ad92359fdb21","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:38.968698738Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1541,"session_id":"1c9476dc-61e8-49a6-9c9a-ad92359fdb21"} {"msg":"Upstairs opts: Upstairs UUID: 59470ed2-b8ec-48d2-84c1-3ddc35dd7852, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:38.968721427Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1541,"session_id":"1c9476dc-61e8-49a6-9c9a-ad92359fdb21"} {"msg":"Crucible stats registered with UUID: 59470ed2-b8ec-48d2-84c1-3ddc35dd7852","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:38.968731347Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1541,"session_id":"1c9476dc-61e8-49a6-9c9a-ad92359fdb21"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:45:38.968759296Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1541,"session_id":"1c9476dc-61e8-49a6-9c9a-ad92359fdb21"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:38.968911032Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1541,"":"io task","client":"0","":"downstairs","session_id":"1c9476dc-61e8-49a6-9c9a-ad92359fdb21"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:38.968943551Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1541,"":"io task","client":"2","":"downstairs","session_id":"1c9476dc-61e8-49a6-9c9a-ad92359fdb21"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:38.96895689Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1541,"":"io task","client":"1","":"downstairs","session_id":"1c9476dc-61e8-49a6-9c9a-ad92359fdb21"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:38.969097866Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1541,"task":"crutest"} {"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:38.969121525Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1541} {"msg":"59470ed2-b8ec-48d2-84c1-3ddc35dd7852 active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:38.969130435Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1541,"session_id":"1c9476dc-61e8-49a6-9c9a-ad92359fdb21"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:38.969139505Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1541,"client":"0","":"downstairs","session_id":"1c9476dc-61e8-49a6-9c9a-ad92359fdb21"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:38.969149605Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1541,"client":"1","":"downstairs","session_id":"1c9476dc-61e8-49a6-9c9a-ad92359fdb21"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:38.969159184Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1541,"client":"2","":"downstairs","session_id":"1c9476dc-61e8-49a6-9c9a-ad92359fdb21"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:38.969168614Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1541,"":"io task","client":"2","":"downstairs","session_id":"1c9476dc-61e8-49a6-9c9a-ad92359fdb21"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:38.969178704Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1541,"":"io task","client":"0","":"downstairs","session_id":"1c9476dc-61e8-49a6-9c9a-ad92359fdb21"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:38.969207093Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1541,"":"io task","client":"1","":"downstairs","session_id":"1c9476dc-61e8-49a6-9c9a-ad92359fdb21"} {"msg":"ds_connection connected from Ok(127.0.0.1:61771)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:38.969231382Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1541,"":"io task","client":"2","":"downstairs","session_id":"1c9476dc-61e8-49a6-9c9a-ad92359fdb21"} {"msg":"ds_connection connected from Ok(127.0.0.1:33146)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:38.969242642Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1541,"":"io task","client":"1","":"downstairs","session_id":"1c9476dc-61e8-49a6-9c9a-ad92359fdb21"} {"msg":"ds_connection connected from Ok(127.0.0.1:51962)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:38.969253012Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1541,"":"io task","client":"0","":"downstairs","session_id":"1c9476dc-61e8-49a6-9c9a-ad92359fdb21"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:38.969863144Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1541,"client":"0","":"downstairs","session_id":"1c9476dc-61e8-49a6-9c9a-ad92359fdb21"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:38.969938262Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1541,"client":"2","":"downstairs","session_id":"1c9476dc-61e8-49a6-9c9a-ad92359fdb21"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:38.969950122Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1541,"client":"1","":"downstairs","session_id":"1c9476dc-61e8-49a6-9c9a-ad92359fdb21"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:38.970150326Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1541,"session_id":"1c9476dc-61e8-49a6-9c9a-ad92359fdb21"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:38.970222574Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1541,"session_id":"1c9476dc-61e8-49a6-9c9a-ad92359fdb21"} {"msg":"[0]R flush_numbers[0..12]: [163, 168, 165, 156, 157, 167, 143, 165, 167, 167, 149, 158]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:38.970261213Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1541,"":"downstairs","":"downstairs","session_id":"1c9476dc-61e8-49a6-9c9a-ad92359fdb21"} {"msg":"[0]R generation[0..12]: [98, 100, 99, 95, 96, 100, 88, 99, 100, 100, 92, 96]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:38.970272902Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1541,"":"downstairs","":"downstairs","session_id":"1c9476dc-61e8-49a6-9c9a-ad92359fdb21"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:38.970283952Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1541,"":"downstairs","":"downstairs","session_id":"1c9476dc-61e8-49a6-9c9a-ad92359fdb21"} {"msg":"[1]R flush_numbers[0..12]: [163, 168, 165, 156, 157, 167, 143, 165, 167, 167, 149, 158]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:38.970294602Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1541,"":"downstairs","":"downstairs","session_id":"1c9476dc-61e8-49a6-9c9a-ad92359fdb21"} {"msg":"[1]R generation[0..12]: [98, 100, 99, 95, 96, 100, 88, 99, 100, 100, 92, 96]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:38.970304971Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1541,"":"downstairs","":"downstairs","session_id":"1c9476dc-61e8-49a6-9c9a-ad92359fdb21"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:38.970315781Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1541,"":"downstairs","":"downstairs","session_id":"1c9476dc-61e8-49a6-9c9a-ad92359fdb21"} {"msg":"[2]R flush_numbers[0..12]: [163, 168, 165, 156, 157, 167, 143, 165, 167, 167, 149, 158]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:38.970326401Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1541,"":"downstairs","":"downstairs","session_id":"1c9476dc-61e8-49a6-9c9a-ad92359fdb21"} {"msg":"[2]R generation[0..12]: [98, 100, 99, 95, 96, 100, 88, 99, 100, 100, 92, 96]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:38.97033693Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1541,"":"downstairs","":"downstairs","session_id":"1c9476dc-61e8-49a6-9c9a-ad92359fdb21"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:38.97034803Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1541,"":"downstairs","":"downstairs","session_id":"1c9476dc-61e8-49a6-9c9a-ad92359fdb21"} {"msg":"Max found gen is 102","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:38.97035736Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1541,"":"downstairs","":"downstairs","session_id":"1c9476dc-61e8-49a6-9c9a-ad92359fdb21"} {"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:38.97036706Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1541,"":"downstairs","":"downstairs","session_id":"1c9476dc-61e8-49a6-9c9a-ad92359fdb21"} {"msg":"Next flush: 170","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:38.970376389Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1541,"":"downstairs","":"downstairs","session_id":"1c9476dc-61e8-49a6-9c9a-ad92359fdb21"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:38.970388409Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1541,"":"downstairs","":"downstairs","session_id":"1c9476dc-61e8-49a6-9c9a-ad92359fdb21"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:38.970398139Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1541,"session_id":"1c9476dc-61e8-49a6-9c9a-ad92359fdb21"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:38.970406768Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1541,"client":"0","":"downstairs","session_id":"1c9476dc-61e8-49a6-9c9a-ad92359fdb21"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:38.970416328Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1541,"client":"1","":"downstairs","session_id":"1c9476dc-61e8-49a6-9c9a-ad92359fdb21"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:38.970425868Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1541,"client":"2","":"downstairs","session_id":"1c9476dc-61e8-49a6-9c9a-ad92359fdb21"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:38.970435528Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1541,"session_id":"1c9476dc-61e8-49a6-9c9a-ad92359fdb21"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:38.970444657Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1541,"session_id":"1c9476dc-61e8-49a6-9c9a-ad92359fdb21"} {"msg":"59470ed2-b8ec-48d2-84c1-3ddc35dd7852 is now active with session: 1c9476dc-61e8-49a6-9c9a-ad92359fdb21","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:38.970454407Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1541,"session_id":"1c9476dc-61e8-49a6-9c9a-ad92359fdb21"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:38.970463027Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1541,"session_id":"1c9476dc-61e8-49a6-9c9a-ad92359fdb21"} {"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:38.970472067Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1541} {"msg":"Activated sub_volume 59470ed2-b8ec-48d2-84c1-3ddc35dd7852","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:38.970479976Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1541} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:38.970493056Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1541,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:38.970749719Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1541,"task":"crutest"} 01/30 Write block 338 len 36864 data: 7 6 6 8 7 8 8 8 8 02/30 Read block 452 len 12288 03/30 Write block 324 len 24576 data: 12 11 12 13 11 12 04/30 Read block 8 len 40960 05/30 Read block 381 len 12288 06/30 Write block 206 len 36864 data: 11 9 11 12 12 10 11 12 11 07/30 Read block 520 len 24576 08/30 Write block 355 len 20480 data: 7 8 8 6 5 09/30 Read block 149 len 32768 10/30 Write block 233 len 24576 data: 9 10 7 6 6 5 11/30 Read block 407 len 12288 12/30 Read block 482 len 16384 13/30 Read block 327 len 8192 14/30 Read block 283 len 12288 15/30 Read block 10 len 36864 16/30 Read block 548 len 16384 17/30 Read block 267 len 4096 18/30 Write block 70 len 20480 data: 8 8 7 8 8 19/30 Flush 20/30 Write block 496 len 36864 data: 9 5 7 8 9 8 8 9 9 21/30 Read block 478 len 8192 22/30 Read block 144 len 32768 23/30 Write block 184 len 40960 data: 11 10 11 10 8 8 7 8 8 7 24/30 Write block 382 len 8192 data: 13 10 25/30 Write block 72 len 32768 data: 8 9 9 9 9 9 11 10 26/30 Write block 225 len 4096 data: 13 27/30 Read block 109 len 12288 28/30 Read block 420 len 12288 29/30 Read block 454 len 24576 30/30 Read block 367 len 12288 ---------------------------------------------------------------- Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Read 3 Done Done Done false 1002 Acked Write 6 Done Done Done false 1003 Acked Read 10 Done Done Done false 1004 Acked Read 3 Done Done Done false 1005 Acked Write 9 Done Done Done false 1006 Acked Read 6 Done Done Done false 1007 Acked Write 5 Done Done Done false 1008 Acked Read 8 Done Sent Done false 1009 Acked Write 6 Done Sent Done false 1010 Acked Read 3 Done Sent Done false 1011 Acked Read 4 Done Sent Done false 1012 Acked Read 2 Done Sent Done false 1013 Acked Read 3 Done Sent Done false 1014 Acked Read 9 Done Sent Done false 1015 Acked Read 4 Done Sent Done false 1016 Acked Read 1 Done Sent Done false 1017 Acked Write 5 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Write 9 Done Sent Done false 1020 Acked Read 2 Done Sent Done false 1021 Acked Read 8 Done Sent Done false 1022 Acked Write 10 Done Sent Done false 1023 Acked Write 2 Done Sent Done false 1024 Acked Write 8 Done Sent Done false 1025 Acked Write 1 Done Sent Done false 1026 Acked Read 3 Done Sent Done false 1027 Acked Read 3 Done Sent Done false 1028 Acked Read 6 Done Sent Done false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 22 1 23 Done 30 8 29 67 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) None Some(JobId(1018)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:45:38.979961473Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1541} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:38.980158007Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1541,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 163 163 163 F F F 1 020-039 100 100 100 168 168 168 F F F 2 040-059 99 99 99 165 165 165 F F F 3 060-079 102 95 102 170 156 170 T F T <--- 4 080-099 96 96 96 157 157 157 F F F 5 100-119 100 100 100 167 167 167 F F F 6 120-139 88 88 88 143 143 143 F F F 7 140-159 99 99 99 165 165 165 F F F 8 160-179 100 100 100 167 167 167 F F F 9 180-199 100 100 100 167 167 167 T F T <--- 10 200-219 102 92 102 170 149 170 F T F <--- 11 220-239 102 96 102 170 158 170 T F T <--- 12 240-259 95 95 95 156 156 156 F F F 13 260-279 88 88 88 143 143 143 F F F 14 280-299 100 100 100 168 168 168 F F F 15 300-319 100 100 100 168 168 168 F F F 16 320-339 102 101 102 170 169 170 F T F <--- 17 340-359 102 100 102 170 167 170 F T F <--- 18 360-379 100 100 100 167 167 167 F F F 19 380-399 101 101 101 169 169 169 T F T <--- 20 400-419 101 101 101 169 169 169 F F F 21 420-439 99 99 99 165 165 165 F F F 22 440-459 100 100 100 166 166 166 F F F 23 460-479 100 100 100 166 166 166 F F F 24 480-499 98 98 98 161 161 161 T F T <--- 25 500-519 100 100 100 168 168 168 T F T <--- 26 520-539 96 96 96 157 157 157 F F F 27 540-559 92 92 92 149 149 149 F F F 28 560-579 92 92 92 148 148 148 F F F 29 580-599 101 101 101 169 169 169 F F F Max gen: 102, Max flush: 170 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 51 Verifying data now No change after verify Loop: 51 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 163 163 163 F F F 1 020-039 100 100 100 168 168 168 F F F 2 040-059 99 99 99 165 165 165 F F F 3 060-079 103 103 103 171 171 171 F F F 4 080-099 96 96 96 157 157 157 F F F 5 100-119 100 100 100 167 167 167 F F F 6 120-139 88 88 88 143 143 143 F F F 7 140-159 99 99 99 165 165 165 F F F 8 160-179 100 100 100 167 167 167 F F F 9 180-199 103 103 103 171 171 171 F F F 10 200-219 102 102 102 170 170 170 F F F 11 220-239 103 103 103 171 171 171 F F F 12 240-259 95 95 95 156 156 156 F F F 13 260-279 88 88 88 143 143 143 F F F 14 280-299 100 100 100 168 168 168 F F F 15 300-319 100 100 100 168 168 168 F F F 16 320-339 102 102 102 170 170 170 F F F 17 340-359 102 102 102 170 170 170 F F F 18 360-379 100 100 100 167 167 167 F F F 19 380-399 103 103 103 171 171 171 F F F 20 400-419 101 101 101 169 169 169 F F F 21 420-439 99 99 99 165 165 165 F F F 22 440-459 100 100 100 166 166 166 F F F 23 460-479 100 100 100 166 166 166 F F F 24 480-499 103 103 103 171 171 171 F F F 25 500-519 103 103 103 171 171 171 F F F 26 520-539 96 96 96 157 157 157 F F F 27 540-559 92 92 92 149 149 149 F F F 28 560-579 92 92 92 148 148 148 F F F 29 580-599 101 101 101 169 169 169 F F F Max gen: 103, Max flush: 171 Begin loop 52 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:43.219457166Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1549,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:45:43.226445435Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1549,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:43.226500053Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1549,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:43.233304217Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1549,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:43.233581099Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1549} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:43.233606498Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1549} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:43.233615018Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1549} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:43.233626387Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1549} {"msg":"Crucible c0ec7ffd-c4ac-4a4e-88d0-73c3ff59c4cc has session id: 8fc783ba-f070-4393-97e1-3b5a418e43e9","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:43.23389204Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1549,"session_id":"8fc783ba-f070-4393-97e1-3b5a418e43e9"} {"msg":"Upstairs opts: Upstairs UUID: c0ec7ffd-c4ac-4a4e-88d0-73c3ff59c4cc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:43.233960258Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1549,"session_id":"8fc783ba-f070-4393-97e1-3b5a418e43e9"} {"msg":"Crucible stats registered with UUID: c0ec7ffd-c4ac-4a4e-88d0-73c3ff59c4cc","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:43.233970658Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1549,"session_id":"8fc783ba-f070-4393-97e1-3b5a418e43e9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:45:43.233979597Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1549,"session_id":"8fc783ba-f070-4393-97e1-3b5a418e43e9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:43.234117023Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1549,"":"io task","client":"2","":"downstairs","session_id":"8fc783ba-f070-4393-97e1-3b5a418e43e9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:43.234134253Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1549,"":"io task","client":"0","":"downstairs","session_id":"8fc783ba-f070-4393-97e1-3b5a418e43e9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:43.234144483Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1549,"":"io task","client":"1","":"downstairs","session_id":"8fc783ba-f070-4393-97e1-3b5a418e43e9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:43.234300248Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1549,"task":"crutest"} {"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:43.234311798Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1549} {"msg":"c0ec7ffd-c4ac-4a4e-88d0-73c3ff59c4cc active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:43.234324947Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1549,"session_id":"8fc783ba-f070-4393-97e1-3b5a418e43e9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:43.234335407Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1549,"client":"0","":"downstairs","session_id":"8fc783ba-f070-4393-97e1-3b5a418e43e9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:43.234345107Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1549,"client":"1","":"downstairs","session_id":"8fc783ba-f070-4393-97e1-3b5a418e43e9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:43.234354716Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1549,"client":"2","":"downstairs","session_id":"8fc783ba-f070-4393-97e1-3b5a418e43e9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:43.234364216Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1549,"":"io task","client":"2","":"downstairs","session_id":"8fc783ba-f070-4393-97e1-3b5a418e43e9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:43.234374146Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1549,"":"io task","client":"0","":"downstairs","session_id":"8fc783ba-f070-4393-97e1-3b5a418e43e9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:43.234384016Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1549,"":"io task","client":"1","":"downstairs","session_id":"8fc783ba-f070-4393-97e1-3b5a418e43e9"} {"msg":"ds_connection connected from Ok(127.0.0.1:61549)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:43.234460463Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1549,"":"io task","client":"2","":"downstairs","session_id":"8fc783ba-f070-4393-97e1-3b5a418e43e9"} {"msg":"ds_connection connected from Ok(127.0.0.1:49865)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:43.234485813Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1549,"":"io task","client":"0","":"downstairs","session_id":"8fc783ba-f070-4393-97e1-3b5a418e43e9"} {"msg":"ds_connection connected from Ok(127.0.0.1:55468)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:43.234506062Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1549,"":"io task","client":"1","":"downstairs","session_id":"8fc783ba-f070-4393-97e1-3b5a418e43e9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:43.235189002Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1549,"client":"1","":"downstairs","session_id":"8fc783ba-f070-4393-97e1-3b5a418e43e9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:43.235215722Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1549,"client":"2","":"downstairs","session_id":"8fc783ba-f070-4393-97e1-3b5a418e43e9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:43.235293729Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1549,"client":"0","":"downstairs","session_id":"8fc783ba-f070-4393-97e1-3b5a418e43e9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:43.235496983Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1549,"session_id":"8fc783ba-f070-4393-97e1-3b5a418e43e9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:43.235506433Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1549,"session_id":"8fc783ba-f070-4393-97e1-3b5a418e43e9"} {"msg":"[0]R flush_numbers[0..12]: [163, 168, 165, 171, 157, 167, 143, 165, 167, 171, 170, 171]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:43.23560455Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1549,"":"downstairs","":"downstairs","session_id":"8fc783ba-f070-4393-97e1-3b5a418e43e9"} {"msg":"[0]R generation[0..12]: [98, 100, 99, 103, 96, 100, 88, 99, 100, 103, 102, 103]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:43.23561797Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1549,"":"downstairs","":"downstairs","session_id":"8fc783ba-f070-4393-97e1-3b5a418e43e9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:43.23562903Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1549,"":"downstairs","":"downstairs","session_id":"8fc783ba-f070-4393-97e1-3b5a418e43e9"} {"msg":"[1]R flush_numbers[0..12]: [163, 168, 165, 171, 157, 167, 143, 165, 167, 171, 170, 171]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:43.235639759Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1549,"":"downstairs","":"downstairs","session_id":"8fc783ba-f070-4393-97e1-3b5a418e43e9"} {"msg":"[1]R generation[0..12]: [98, 100, 99, 103, 96, 100, 88, 99, 100, 103, 102, 103]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:43.235650229Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1549,"":"downstairs","":"downstairs","session_id":"8fc783ba-f070-4393-97e1-3b5a418e43e9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:43.235661189Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1549,"":"downstairs","":"downstairs","session_id":"8fc783ba-f070-4393-97e1-3b5a418e43e9"} {"msg":"[2]R flush_numbers[0..12]: [163, 168, 165, 171, 157, 167, 143, 165, 167, 171, 170, 171]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:43.235671658Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1549,"":"downstairs","":"downstairs","session_id":"8fc783ba-f070-4393-97e1-3b5a418e43e9"} {"msg":"[2]R generation[0..12]: [98, 100, 99, 103, 96, 100, 88, 99, 100, 103, 102, 103]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:43.235682008Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1549,"":"downstairs","":"downstairs","session_id":"8fc783ba-f070-4393-97e1-3b5a418e43e9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:43.235692858Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1549,"":"downstairs","":"downstairs","session_id":"8fc783ba-f070-4393-97e1-3b5a418e43e9"} {"msg":"Max found gen is 104","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:43.235702288Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1549,"":"downstairs","":"downstairs","session_id":"8fc783ba-f070-4393-97e1-3b5a418e43e9"} {"msg":"Generation requested: 104 >= found:104","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:43.235711877Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1549,"":"downstairs","":"downstairs","session_id":"8fc783ba-f070-4393-97e1-3b5a418e43e9"} {"msg":"Next flush: 172","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:43.235721157Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1549,"":"downstairs","":"downstairs","session_id":"8fc783ba-f070-4393-97e1-3b5a418e43e9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:43.235730487Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1549,"":"downstairs","":"downstairs","session_id":"8fc783ba-f070-4393-97e1-3b5a418e43e9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:43.235740086Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1549,"session_id":"8fc783ba-f070-4393-97e1-3b5a418e43e9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:43.235748786Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1549,"client":"0","":"downstairs","session_id":"8fc783ba-f070-4393-97e1-3b5a418e43e9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:43.235758266Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1549,"client":"1","":"downstairs","session_id":"8fc783ba-f070-4393-97e1-3b5a418e43e9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:43.235768936Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1549,"client":"2","":"downstairs","session_id":"8fc783ba-f070-4393-97e1-3b5a418e43e9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:43.235778685Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1549,"session_id":"8fc783ba-f070-4393-97e1-3b5a418e43e9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:43.235787725Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1549,"session_id":"8fc783ba-f070-4393-97e1-3b5a418e43e9"} {"msg":"c0ec7ffd-c4ac-4a4e-88d0-73c3ff59c4cc is now active with session: 8fc783ba-f070-4393-97e1-3b5a418e43e9","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:43.235797505Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1549,"session_id":"8fc783ba-f070-4393-97e1-3b5a418e43e9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:43.235806095Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1549,"session_id":"8fc783ba-f070-4393-97e1-3b5a418e43e9"} {"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:43.235815184Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1549} {"msg":"Activated sub_volume c0ec7ffd-c4ac-4a4e-88d0-73c3ff59c4cc","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:43.235823114Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1549} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:43.235831114Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1549,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:43.236130115Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1549,"task":"crutest"} 01/30 Write block 557 len 8192 data: 7 10 02/30 Write block 399 len 36864 data: 7 7 7 8 7 8 7 6 7 03/30 Read block 75 len 32768 04/30 Read block 89 len 16384 05/30 Flush 06/30 Write block 548 len 28672 data: 10 10 11 7 8 9 6 07/30 Read block 498 len 32768 08/30 Flush 09/30 Write block 50 len 20480 data: 8 6 6 5 7 10/30 Write block 276 len 24576 data: 5 6 6 7 6 7 11/30 Read block 380 len 16384 12/30 Write block 476 len 8192 data: 12 11 13/30 Read block 518 len 28672 14/30 Read block 134 len 28672 15/30 Write block 416 len 16384 data: 8 8 9 10 16/30 Write block 367 len 4096 data: 11 17/30 Flush 18/30 Write block 578 len 24576 data: 10 8 8 7 9 9 19/30 Read block 414 len 8192 20/30 Write block 492 len 4096 data: 9 21/30 Read block 576 len 40960 22/30 Read block 218 len 40960 23/30 Read block 106 len 4096 24/30 Read block 98 len 28672 25/30 Read block 228 len 40960 26/30 Write block 27 len 4096 data: 4 27/30 Read block 101 len 4096 28/30 Write block 112 len 32768 data: 9 9 8 8 9 9 8 10 29/30 Read block 323 len 8192 30/30 Read block 232 len 40960 ---------------------------------------------------------------- Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Write 9 Done Done Done false 1002 Acked Read 8 Done Done Done false 1003 Acked Read 4 Sent Done Done false 1004 Acked Flush 0 Sent Done Done false 1005 Acked Write 7 Sent Done Done false 1006 Acked Read 8 Sent Done Done false 1007 Acked Flush 0 Sent Done Done false 1008 Acked Write 5 Sent Done Done false 1009 Acked Write 6 Sent Done Done false 1010 Acked Read 4 Sent Done Done false 1011 Acked Write 2 Sent Done Done false 1012 Acked Read 7 Sent Done Done false 1013 Acked Read 7 Sent Done Done false 1014 Acked Write 4 Sent Done Done false 1015 Acked Write 1 Sent Done Done false 1016 Acked Flush 0 Sent Done Done false 1017 Acked Write 6 Sent Done Done false 1018 Acked Read 2 Sent Done Done false 1019 Acked Write 1 Sent Done Done false 1020 Acked Read 10 Sent Done Done false 1021 Acked Read 10 Sent Done Done false 1022 Acked Read 1 Sent Done Done false 1023 Acked Read 7 Sent Done Done false 1024 Acked Read 10 Sent Done Done false 1025 Acked Write 1 Sent Done Done false 1026 Acked Read 1 Sent Done Done false 1027 Acked Write 8 Sent Done Done false 1028 Acked Read 2 Sent Done Done false 1029 Acked Read 10 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 27 0 0 27 Done 3 30 30 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:45:43.244458465Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1549} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:43.24462554Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1549,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 163 163 163 F F F 1 020-039 100 100 100 168 168 168 F T T <--- 2 040-059 99 104 104 165 174 174 F F F <--- 3 060-079 103 103 103 171 171 171 F F F 4 080-099 96 96 96 157 157 157 F F F 5 100-119 100 100 100 167 167 167 F T T <--- 6 120-139 88 88 88 143 143 143 F F F 7 140-159 99 99 99 165 165 165 F F F 8 160-179 100 100 100 167 167 167 F F F 9 180-199 103 103 103 171 171 171 F F F 10 200-219 102 102 102 170 170 170 F F F 11 220-239 103 103 103 171 171 171 F F F 12 240-259 95 95 95 156 156 156 F F F 13 260-279 88 104 104 143 174 174 F F F <--- 14 280-299 100 104 104 168 174 174 F F F <--- 15 300-319 100 100 100 168 168 168 F F F 16 320-339 102 102 102 170 170 170 F F F 17 340-359 102 102 102 170 170 170 F F F 18 360-379 100 104 104 167 174 174 F F F <--- 19 380-399 103 104 104 171 172 172 T F F <--- 20 400-419 101 104 104 169 174 174 T F F <--- 21 420-439 99 99 99 165 165 165 F F F 22 440-459 100 100 100 166 166 166 F F F 23 460-479 100 104 104 166 174 174 F F F <--- 24 480-499 103 103 103 171 171 171 F T T <--- 25 500-519 103 103 103 171 171 171 F F F 26 520-539 96 96 96 157 157 157 F F F 27 540-559 92 104 104 149 173 173 T F F <--- 28 560-579 92 92 92 148 148 148 F T T <--- 29 580-599 101 101 101 169 169 169 F T T <--- Max gen: 104, Max flush: 174 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 52 Verifying data now No change after verify Loop: 52 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 163 163 163 F F F 1 020-039 105 105 105 175 175 175 F F F 2 040-059 104 104 104 174 174 174 F F F 3 060-079 103 103 103 171 171 171 F F F 4 080-099 96 96 96 157 157 157 F F F 5 100-119 105 105 105 175 175 175 F F F 6 120-139 88 88 88 143 143 143 F F F 7 140-159 99 99 99 165 165 165 F F F 8 160-179 100 100 100 167 167 167 F F F 9 180-199 103 103 103 171 171 171 F F F 10 200-219 102 102 102 170 170 170 F F F 11 220-239 103 103 103 171 171 171 F F F 12 240-259 95 95 95 156 156 156 F F F 13 260-279 104 104 104 174 174 174 F F F 14 280-299 104 104 104 174 174 174 F F F 15 300-319 100 100 100 168 168 168 F F F 16 320-339 102 102 102 170 170 170 F F F 17 340-359 102 102 102 170 170 170 F F F 18 360-379 104 104 104 174 174 174 F F F 19 380-399 104 104 104 172 172 172 F F F 20 400-419 104 104 104 174 174 174 F F F 21 420-439 99 99 99 165 165 165 F F F 22 440-459 100 100 100 166 166 166 F F F 23 460-479 104 104 104 174 174 174 F F F 24 480-499 105 105 105 175 175 175 F F F 25 500-519 103 103 103 171 171 171 F F F 26 520-539 96 96 96 157 157 157 F F F 27 540-559 104 104 104 173 173 173 F F F 28 560-579 105 105 105 175 175 175 F F F 29 580-599 105 105 105 175 175 175 F F F Max gen: 105, Max flush: 175 Begin loop 53 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:47.498345133Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1557,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:47.506501038Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1557,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:47.506809919Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1557} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:47.506839048Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1557} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:47.506847618Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1557} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:47.506858958Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1557} {"msg":"Crucible 6cb17c48-8174-4b59-91c3-0876c13311de has session id: 78c6a50b-19f3-4155-b26d-ea6745baac5c","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:47.507209018Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1557,"session_id":"78c6a50b-19f3-4155-b26d-ea6745baac5c"} {"msg":"Upstairs opts: Upstairs UUID: 6cb17c48-8174-4b59-91c3-0876c13311de, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:47.507233217Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1557,"session_id":"78c6a50b-19f3-4155-b26d-ea6745baac5c"} {"msg":"Crucible stats registered with UUID: 6cb17c48-8174-4b59-91c3-0876c13311de","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:47.507242877Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1557,"session_id":"78c6a50b-19f3-4155-b26d-ea6745baac5c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:45:47.507251966Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1557,"session_id":"78c6a50b-19f3-4155-b26d-ea6745baac5c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:47.507412552Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1557,"":"io task","client":"2","":"downstairs","session_id":"78c6a50b-19f3-4155-b26d-ea6745baac5c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:47.507447131Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1557,"":"io task","client":"0","":"downstairs","session_id":"78c6a50b-19f3-4155-b26d-ea6745baac5c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:47.50745816Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1557,"":"io task","client":"1","":"downstairs","session_id":"78c6a50b-19f3-4155-b26d-ea6745baac5c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:47.507596276Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1557,"task":"crutest"} {"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:47.507611156Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1557} {"msg":"6cb17c48-8174-4b59-91c3-0876c13311de active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:47.507644455Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1557,"session_id":"78c6a50b-19f3-4155-b26d-ea6745baac5c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:47.507653955Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1557,"client":"0","":"downstairs","session_id":"78c6a50b-19f3-4155-b26d-ea6745baac5c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:47.507663784Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1557,"client":"1","":"downstairs","session_id":"78c6a50b-19f3-4155-b26d-ea6745baac5c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:47.507673434Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1557,"client":"2","":"downstairs","session_id":"78c6a50b-19f3-4155-b26d-ea6745baac5c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:47.507682964Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1557,"":"io task","client":"2","":"downstairs","session_id":"78c6a50b-19f3-4155-b26d-ea6745baac5c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:47.507692874Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1557,"":"io task","client":"0","":"downstairs","session_id":"78c6a50b-19f3-4155-b26d-ea6745baac5c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:47.507725833Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1557,"":"io task","client":"1","":"downstairs","session_id":"78c6a50b-19f3-4155-b26d-ea6745baac5c"} {"msg":"ds_connection connected from Ok(127.0.0.1:65460)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:47.507752852Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1557,"":"io task","client":"2","":"downstairs","session_id":"78c6a50b-19f3-4155-b26d-ea6745baac5c"} {"msg":"ds_connection connected from Ok(127.0.0.1:52679)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:47.507768641Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1557,"":"io task","client":"1","":"downstairs","session_id":"78c6a50b-19f3-4155-b26d-ea6745baac5c"} {"msg":"ds_connection connected from Ok(127.0.0.1:33577)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:47.507779021Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1557,"":"io task","client":"0","":"downstairs","session_id":"78c6a50b-19f3-4155-b26d-ea6745baac5c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:47.508448992Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1557,"client":"1","":"downstairs","session_id":"78c6a50b-19f3-4155-b26d-ea6745baac5c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:47.508468881Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1557,"client":"2","":"downstairs","session_id":"78c6a50b-19f3-4155-b26d-ea6745baac5c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:47.508479741Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1557,"client":"0","":"downstairs","session_id":"78c6a50b-19f3-4155-b26d-ea6745baac5c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:47.508805911Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1557,"session_id":"78c6a50b-19f3-4155-b26d-ea6745baac5c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:47.508815471Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1557,"session_id":"78c6a50b-19f3-4155-b26d-ea6745baac5c"} {"msg":"[0]R flush_numbers[0..12]: [163, 175, 174, 171, 157, 175, 143, 165, 167, 171, 170, 171]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:47.50885473Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1557,"":"downstairs","":"downstairs","session_id":"78c6a50b-19f3-4155-b26d-ea6745baac5c"} {"msg":"[0]R generation[0..12]: [98, 105, 104, 103, 96, 105, 88, 99, 100, 103, 102, 103]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:47.50886593Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1557,"":"downstairs","":"downstairs","session_id":"78c6a50b-19f3-4155-b26d-ea6745baac5c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:47.508876999Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1557,"":"downstairs","":"downstairs","session_id":"78c6a50b-19f3-4155-b26d-ea6745baac5c"} {"msg":"[1]R flush_numbers[0..12]: [163, 175, 174, 171, 157, 175, 143, 165, 167, 171, 170, 171]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:47.508887739Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1557,"":"downstairs","":"downstairs","session_id":"78c6a50b-19f3-4155-b26d-ea6745baac5c"} {"msg":"[1]R generation[0..12]: [98, 105, 104, 103, 96, 105, 88, 99, 100, 103, 102, 103]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:47.508898399Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1557,"":"downstairs","":"downstairs","session_id":"78c6a50b-19f3-4155-b26d-ea6745baac5c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:47.508909299Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1557,"":"downstairs","":"downstairs","session_id":"78c6a50b-19f3-4155-b26d-ea6745baac5c"} {"msg":"[2]R flush_numbers[0..12]: [163, 175, 174, 171, 157, 175, 143, 165, 167, 171, 170, 171]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:47.508921958Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1557,"":"downstairs","":"downstairs","session_id":"78c6a50b-19f3-4155-b26d-ea6745baac5c"} {"msg":"[2]R generation[0..12]: [98, 105, 104, 103, 96, 105, 88, 99, 100, 103, 102, 103]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:47.508932458Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1557,"":"downstairs","":"downstairs","session_id":"78c6a50b-19f3-4155-b26d-ea6745baac5c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:47.508943348Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1557,"":"downstairs","":"downstairs","session_id":"78c6a50b-19f3-4155-b26d-ea6745baac5c"} {"msg":"Max found gen is 106","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:47.508952837Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1557,"":"downstairs","":"downstairs","session_id":"78c6a50b-19f3-4155-b26d-ea6745baac5c"} {"msg":"Generation requested: 106 >= found:106","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:47.508962617Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1557,"":"downstairs","":"downstairs","session_id":"78c6a50b-19f3-4155-b26d-ea6745baac5c"} {"msg":"Next flush: 176","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:47.508972047Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1557,"":"downstairs","":"downstairs","session_id":"78c6a50b-19f3-4155-b26d-ea6745baac5c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:47.508981366Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1557,"":"downstairs","":"downstairs","session_id":"78c6a50b-19f3-4155-b26d-ea6745baac5c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:47.508991106Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1557,"session_id":"78c6a50b-19f3-4155-b26d-ea6745baac5c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:47.508999756Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1557,"client":"0","":"downstairs","session_id":"78c6a50b-19f3-4155-b26d-ea6745baac5c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:47.509009576Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1557,"client":"1","":"downstairs","session_id":"78c6a50b-19f3-4155-b26d-ea6745baac5c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:47.509019195Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1557,"client":"2","":"downstairs","session_id":"78c6a50b-19f3-4155-b26d-ea6745baac5c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:47.509029045Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1557,"session_id":"78c6a50b-19f3-4155-b26d-ea6745baac5c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:47.509038265Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1557,"session_id":"78c6a50b-19f3-4155-b26d-ea6745baac5c"} {"msg":"6cb17c48-8174-4b59-91c3-0876c13311de is now active with session: 78c6a50b-19f3-4155-b26d-ea6745baac5c","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:47.509048164Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1557,"session_id":"78c6a50b-19f3-4155-b26d-ea6745baac5c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:47.509056834Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1557,"session_id":"78c6a50b-19f3-4155-b26d-ea6745baac5c"} {"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:47.509065894Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1557} {"msg":"Activated sub_volume 6cb17c48-8174-4b59-91c3-0876c13311de","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:47.509075384Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1557} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:47.509083423Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1557,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:47.509424244Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1557,"task":"crutest"} 01/30 Write block 391 len 40960 data: 9 10 9 7 10 11 8 6 8 8 02/30 Read block 558 len 4096 03/30 Write block 46 len 32768 data: 6 6 8 7 9 7 7 6 04/30 Flush 05/30 Write block 573 len 4096 data: 9 06/30 Flush 07/30 Write block 158 len 28672 data: 8 12 11 7 6 8 5 08/30 Write block 357 len 32768 data: 9 7 6 7 7 10 11 13 09/30 Read block 307 len 8192 10/30 Write block 110 len 8192 data: 5 7 11/30 Read block 203 len 12288 12/30 Read block 239 len 40960 13/30 Write block 187 len 20480 data: 11 9 9 8 9 14/30 Flush 15/30 Write block 346 len 40960 data: 9 8 7 7 5 3 4 8 8 8 16/30 Write block 314 len 16384 data: 8 7 6 4 17/30 Read block 217 len 12288 18/30 Read block 95 len 36864 19/30 Read block 538 len 28672 20/30 Read block 46 len 20480 21/30 Read block 220 len 24576 22/30 Write block 500 len 28672 data: 10 9 9 10 10 8 8 23/30 Write block 253 len 32768 data: 8 7 11 9 6 6 6 6 24/30 Read block 523 len 4096 25/30 Read block 324 len 12288 26/30 Write block 192 len 24576 data: 9 8 9 8 9 9 27/30 Write block 2 len 4096 data: 6 28/30 Write block 225 len 12288 data: 14 13 14 29/30 Read block 573 len 12288 30/30 Read block 235 len 16384 ---------------------------------------------------------------- Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Sent false 1001 Acked Read 1 Done Done Sent false 1002 Acked Write 8 Done Done Sent false 1003 Acked Flush 0 Done Done Sent false 1004 Acked Write 1 Done Done Sent false 1005 Acked Flush 0 Done Done Sent false 1006 Acked Write 7 Done Done Sent false 1007 Acked Write 8 Done Done Sent false 1008 Acked Read 2 Done Done Sent false 1009 Acked Write 2 Done Done Sent false 1010 Acked Read 3 Done Done Sent false 1011 Acked Read 10 Done Done Sent false 1012 Acked Write 5 Done Done Sent false 1013 Acked Flush 0 Done Done Sent false 1014 Acked Write 10 Done Done Sent false 1015 Acked Write 4 Done Done Sent false 1016 Acked Read 3 Done Done Sent false 1017 Acked Read 9 Done Done Sent false 1018 Acked Read 7 Done Done Sent false 1019 Acked Read 5 Done Done Sent false 1020 Acked Read 6 Done Done Sent false 1021 Acked Write 7 Done Done Sent false 1022 Acked Write 8 Done Done Sent false 1023 Acked Read 1 Done Done Sent false 1024 Acked Read 3 Done Done Sent false 1025 Acked Write 6 Done Done Sent false 1026 Acked Write 1 Done Done Sent false 1027 Acked Write 3 Done Done Sent false 1028 Acked Read 3 Done Done Sent false 1029 Acked Read 4 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 30 30 Done 30 30 0 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1013)) Some(JobId(1013)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:45:47.518162262Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1557} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:47.518329787Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1557,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 163 163 163 T T F <--- 1 020-039 105 105 105 175 175 175 F F F 2 040-059 106 106 104 176 176 174 F F F <--- 3 060-079 103 103 103 171 171 171 F F F 4 080-099 96 96 96 157 157 157 F F F 5 100-119 106 106 105 178 178 175 F F F <--- 6 120-139 88 88 88 143 143 143 F F F 7 140-159 106 106 99 178 178 165 F F F <--- 8 160-179 106 106 100 178 178 167 F F F <--- 9 180-199 106 106 103 178 178 171 T T F <--- 10 200-219 102 102 102 170 170 170 F F F 11 220-239 103 103 103 171 171 171 T T F <--- 12 240-259 95 95 95 156 156 156 T T F <--- 13 260-279 104 104 104 174 174 174 T T F <--- 14 280-299 104 104 104 174 174 174 F F F 15 300-319 100 100 100 168 168 168 T T F <--- 16 320-339 102 102 102 170 170 170 F F F 17 340-359 106 106 102 178 178 170 T T F <--- 18 360-379 106 106 104 178 178 174 F F F <--- 19 380-399 106 106 104 176 176 172 F F F <--- 20 400-419 106 106 104 176 176 174 F F F <--- 21 420-439 99 99 99 165 165 165 F F F 22 440-459 100 100 100 166 166 166 F F F 23 460-479 104 104 104 174 174 174 F F F 24 480-499 105 105 105 175 175 175 F F F 25 500-519 103 103 103 171 171 171 T T F <--- 26 520-539 96 96 96 157 157 157 F F F 27 540-559 104 104 104 173 173 173 F F F 28 560-579 106 106 105 177 177 175 F F F <--- 29 580-599 105 105 105 175 175 175 F F F Max gen: 106, Max flush: 178 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 53 Verifying data now No change after verify Loop: 53 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 107 107 107 179 179 179 F F F 1 020-039 105 105 105 175 175 175 F F F 2 040-059 106 106 106 176 176 176 F F F 3 060-079 103 103 103 171 171 171 F F F 4 080-099 96 96 96 157 157 157 F F F 5 100-119 106 106 106 178 178 178 F F F 6 120-139 88 88 88 143 143 143 F F F 7 140-159 106 106 106 178 178 178 F F F 8 160-179 106 106 106 178 178 178 F F F 9 180-199 107 107 107 179 179 179 F F F 10 200-219 102 102 102 170 170 170 F F F 11 220-239 107 107 107 179 179 179 F F F 12 240-259 107 107 107 179 179 179 F F F 13 260-279 107 107 107 179 179 179 F F F 14 280-299 104 104 104 174 174 174 F F F 15 300-319 107 107 107 179 179 179 F F F 16 320-339 102 102 102 170 170 170 F F F 17 340-359 107 107 107 179 179 179 F F F 18 360-379 106 106 106 178 178 178 F F F 19 380-399 106 106 106 176 176 176 F F F 20 400-419 106 106 106 176 176 176 F F F 21 420-439 99 99 99 165 165 165 F F F 22 440-459 100 100 100 166 166 166 F F F 23 460-479 104 104 104 174 174 174 F F F 24 480-499 105 105 105 175 175 175 F F F 25 500-519 107 107 107 179 179 179 F F F 26 520-539 96 96 96 157 157 157 F F F 27 540-559 104 104 104 173 173 173 F F F 28 560-579 106 106 106 177 177 177 F F F 29 580-599 105 105 105 175 175 175 F F F Max gen: 107, Max flush: 179 Begin loop 54 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:51.770180924Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1565,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:51.778254431Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1565,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:51.778557772Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1565} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:51.778583171Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1565} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:51.778591581Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1565} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:51.778603141Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1565} {"msg":"Crucible 79747fbc-f40f-4852-8fd3-71e2e09f78f4 has session id: ef1d2410-e708-4c2a-a549-b53b6a5fd1d6","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:51.77896966Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1565,"session_id":"ef1d2410-e708-4c2a-a549-b53b6a5fd1d6"} {"msg":"Upstairs opts: Upstairs UUID: 79747fbc-f40f-4852-8fd3-71e2e09f78f4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:51.77899324Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1565,"session_id":"ef1d2410-e708-4c2a-a549-b53b6a5fd1d6"} {"msg":"Crucible stats registered with UUID: 79747fbc-f40f-4852-8fd3-71e2e09f78f4","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:51.779003039Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1565,"session_id":"ef1d2410-e708-4c2a-a549-b53b6a5fd1d6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:45:51.779012989Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1565,"session_id":"ef1d2410-e708-4c2a-a549-b53b6a5fd1d6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:51.779160895Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1565,"":"io task","client":"0","":"downstairs","session_id":"ef1d2410-e708-4c2a-a549-b53b6a5fd1d6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:51.779180614Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1565,"":"io task","client":"1","":"downstairs","session_id":"ef1d2410-e708-4c2a-a549-b53b6a5fd1d6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:51.779190824Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1565,"":"io task","client":"2","":"downstairs","session_id":"ef1d2410-e708-4c2a-a549-b53b6a5fd1d6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:51.77933935Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1565,"task":"crutest"} {"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:51.779354369Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1565} {"msg":"79747fbc-f40f-4852-8fd3-71e2e09f78f4 active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:51.779385498Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1565,"session_id":"ef1d2410-e708-4c2a-a549-b53b6a5fd1d6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:51.779394998Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1565,"client":"0","":"downstairs","session_id":"ef1d2410-e708-4c2a-a549-b53b6a5fd1d6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:51.779404728Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1565,"client":"1","":"downstairs","session_id":"ef1d2410-e708-4c2a-a549-b53b6a5fd1d6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:51.779414367Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1565,"client":"2","":"downstairs","session_id":"ef1d2410-e708-4c2a-a549-b53b6a5fd1d6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:51.779423897Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1565,"":"io task","client":"2","":"downstairs","session_id":"ef1d2410-e708-4c2a-a549-b53b6a5fd1d6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:51.779433837Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1565,"":"io task","client":"0","":"downstairs","session_id":"ef1d2410-e708-4c2a-a549-b53b6a5fd1d6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:51.779443817Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1565,"":"io task","client":"1","":"downstairs","session_id":"ef1d2410-e708-4c2a-a549-b53b6a5fd1d6"} {"msg":"ds_connection connected from Ok(127.0.0.1:63720)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:51.779521124Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1565,"":"io task","client":"0","":"downstairs","session_id":"ef1d2410-e708-4c2a-a549-b53b6a5fd1d6"} {"msg":"ds_connection connected from Ok(127.0.0.1:38056)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:51.779546644Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1565,"":"io task","client":"1","":"downstairs","session_id":"ef1d2410-e708-4c2a-a549-b53b6a5fd1d6"} {"msg":"ds_connection connected from Ok(127.0.0.1:58254)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:51.779557903Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1565,"":"io task","client":"2","":"downstairs","session_id":"ef1d2410-e708-4c2a-a549-b53b6a5fd1d6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:51.780158236Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1565,"client":"0","":"downstairs","session_id":"ef1d2410-e708-4c2a-a549-b53b6a5fd1d6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:51.780201935Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1565,"client":"2","":"downstairs","session_id":"ef1d2410-e708-4c2a-a549-b53b6a5fd1d6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:51.780280812Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1565,"client":"1","":"downstairs","session_id":"ef1d2410-e708-4c2a-a549-b53b6a5fd1d6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:51.780500156Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1565,"session_id":"ef1d2410-e708-4c2a-a549-b53b6a5fd1d6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:51.780523205Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1565,"session_id":"ef1d2410-e708-4c2a-a549-b53b6a5fd1d6"} {"msg":"[0]R flush_numbers[0..12]: [179, 175, 176, 171, 157, 178, 143, 178, 178, 179, 170, 179]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:51.780592733Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1565,"":"downstairs","":"downstairs","session_id":"ef1d2410-e708-4c2a-a549-b53b6a5fd1d6"} {"msg":"[0]R generation[0..12]: [107, 105, 106, 103, 96, 106, 88, 106, 106, 107, 102, 107]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:51.780604423Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1565,"":"downstairs","":"downstairs","session_id":"ef1d2410-e708-4c2a-a549-b53b6a5fd1d6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:51.780620913Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1565,"":"downstairs","":"downstairs","session_id":"ef1d2410-e708-4c2a-a549-b53b6a5fd1d6"} {"msg":"[1]R flush_numbers[0..12]: [179, 175, 176, 171, 157, 178, 143, 178, 178, 179, 170, 179]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:51.780631662Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1565,"":"downstairs","":"downstairs","session_id":"ef1d2410-e708-4c2a-a549-b53b6a5fd1d6"} {"msg":"[1]R generation[0..12]: [107, 105, 106, 103, 96, 106, 88, 106, 106, 107, 102, 107]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:51.780642342Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1565,"":"downstairs","":"downstairs","session_id":"ef1d2410-e708-4c2a-a549-b53b6a5fd1d6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:51.780653222Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1565,"":"downstairs","":"downstairs","session_id":"ef1d2410-e708-4c2a-a549-b53b6a5fd1d6"} {"msg":"[2]R flush_numbers[0..12]: [179, 175, 176, 171, 157, 178, 143, 178, 178, 179, 170, 179]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:51.780663591Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1565,"":"downstairs","":"downstairs","session_id":"ef1d2410-e708-4c2a-a549-b53b6a5fd1d6"} {"msg":"[2]R generation[0..12]: [107, 105, 106, 103, 96, 106, 88, 106, 106, 107, 102, 107]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:51.780674141Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1565,"":"downstairs","":"downstairs","session_id":"ef1d2410-e708-4c2a-a549-b53b6a5fd1d6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:51.780684911Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1565,"":"downstairs","":"downstairs","session_id":"ef1d2410-e708-4c2a-a549-b53b6a5fd1d6"} {"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:51.78069436Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1565,"":"downstairs","":"downstairs","session_id":"ef1d2410-e708-4c2a-a549-b53b6a5fd1d6"} {"msg":"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:51.78070427Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1565,"":"downstairs","":"downstairs","session_id":"ef1d2410-e708-4c2a-a549-b53b6a5fd1d6"} {"msg":"Next flush: 180","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:51.78071603Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1565,"":"downstairs","":"downstairs","session_id":"ef1d2410-e708-4c2a-a549-b53b6a5fd1d6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:51.78072549Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1565,"":"downstairs","":"downstairs","session_id":"ef1d2410-e708-4c2a-a549-b53b6a5fd1d6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:51.780735199Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1565,"session_id":"ef1d2410-e708-4c2a-a549-b53b6a5fd1d6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:51.780743779Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1565,"client":"0","":"downstairs","session_id":"ef1d2410-e708-4c2a-a549-b53b6a5fd1d6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:51.780753569Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1565,"client":"1","":"downstairs","session_id":"ef1d2410-e708-4c2a-a549-b53b6a5fd1d6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:51.780763218Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1565,"client":"2","":"downstairs","session_id":"ef1d2410-e708-4c2a-a549-b53b6a5fd1d6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:51.780772908Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1565,"session_id":"ef1d2410-e708-4c2a-a549-b53b6a5fd1d6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:51.780781998Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1565,"session_id":"ef1d2410-e708-4c2a-a549-b53b6a5fd1d6"} {"msg":"79747fbc-f40f-4852-8fd3-71e2e09f78f4 is now active with session: ef1d2410-e708-4c2a-a549-b53b6a5fd1d6","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:51.780791858Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1565,"session_id":"ef1d2410-e708-4c2a-a549-b53b6a5fd1d6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:51.780800387Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1565,"session_id":"ef1d2410-e708-4c2a-a549-b53b6a5fd1d6"} {"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:51.780809397Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1565} {"msg":"Activated sub_volume 79747fbc-f40f-4852-8fd3-71e2e09f78f4","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:51.780817367Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1565} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:51.780825437Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1565,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:51.781126658Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1565,"task":"crutest"} 01/30 Write block 324 len 16384 data: 13 12 13 14 02/30 Write block 474 len 24576 data: 6 9 13 12 11 10 03/30 Flush 04/30 Write block 319 len 28672 data: 4 3 6 6 10 14 13 05/30 Read block 510 len 8192 06/30 Read block 135 len 24576 07/30 Write block 400 len 32768 data: 9 8 9 8 9 8 7 8 08/30 Write block 500 len 20480 data: 11 10 10 11 11 09/30 Read block 272 len 24576 10/30 Read block 402 len 20480 11/30 Write block 444 len 16384 data: 5 5 5 4 12/30 Write block 151 len 32768 data: 6 5 4 6 5 6 6 9 13/30 Write block 478 len 4096 data: 12 14/30 Read block 55 len 28672 15/30 Flush 16/30 Write block 586 len 36864 data: 9 12 11 7 7 6 5 5 6 17/30 Write block 401 len 20480 data: 9 10 9 10 9 18/30 Write block 55 len 20480 data: 8 8 8 10 8 19/30 Flush 20/30 Write block 10 len 4096 data: 9 21/30 Read block 348 len 24576 22/30 Read block 481 len 16384 23/30 Write block 411 len 20480 data: 11 11 11 8 5 24/30 Write block 503 len 20480 data: 12 12 9 9 10 25/30 Read block 384 len 32768 26/30 Read block 82 len 32768 27/30 Write block 529 len 36864 data: 6 7 7 13 12 12 14 12 14 28/30 Write block 221 len 24576 data: 9 11 12 13 15 14 29/30 Write block 123 len 4096 data: 10 30/30 Read block 535 len 8192 ---------------------------------------------------------------- Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1003 Acked Write 7 Done Done Done false 1004 Acked Read 2 Done Done Done false 1005 Acked Read 6 Done Done Done false 1006 Acked Write 8 Done Done Done false 1007 Acked Write 5 Done Done Done false 1008 Acked Read 6 Done Sent Done false 1009 Acked Read 5 Done Sent Done false 1010 Acked Write 4 Done Sent Done false 1011 Acked Write 8 Done Sent Done false 1012 Acked Write 1 Done Sent Done false 1013 Acked Read 7 Done Sent Done false 1014 Acked Flush 0 Done Sent Done false 1015 Acked Write 9 Done Sent Done false 1016 Acked Write 5 Done Sent Done false 1017 Acked Write 5 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Write 1 Done Sent Done false 1020 Acked Read 6 Done Sent Done false 1021 Acked Read 4 Done Sent Done false 1022 Acked Write 5 Done Sent Done false 1023 Acked Write 5 Done Sent Done false 1024 Acked Read 8 Done Sent Done false 1025 Acked Read 8 Done Sent Done false 1026 Acked Write 9 Done Sent Done false 1027 Acked Write 6 Done Sent Done false 1028 Acked Write 1 Done Sent Done false 1029 Acked Read 2 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 22 0 22 Done 27 5 27 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1002)) Some(JobId(1018)) Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:45:51.789705011Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1565} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:51.789878046Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1565,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 107 107 107 179 179 179 T F T <--- 1 020-039 105 105 105 175 175 175 F F F 2 040-059 108 106 108 182 176 182 F F F <--- 3 060-079 103 103 103 171 171 171 F F F 4 080-099 96 96 96 157 157 157 F F F 5 100-119 106 106 106 178 178 178 F F F 6 120-139 88 88 88 143 143 143 T F T <--- 7 140-159 108 106 108 181 178 181 F F F <--- 8 160-179 106 106 106 178 178 178 F F F 9 180-199 107 107 107 179 179 179 F F F 10 200-219 102 102 102 170 170 170 F F F 11 220-239 107 107 107 179 179 179 T F T <--- 12 240-259 107 107 107 179 179 179 F F F 13 260-279 107 107 107 179 179 179 F F F 14 280-299 104 104 104 174 174 174 F F F 15 300-319 108 107 108 181 179 181 F T F <--- 16 320-339 108 108 108 181 180 181 F T F <--- 17 340-359 107 107 107 179 179 179 F F F 18 360-379 106 106 106 178 178 178 F F F 19 380-399 106 106 106 176 176 176 F F F 20 400-419 108 106 108 182 176 182 T T T <--- 21 420-439 99 99 99 165 165 165 F F F 22 440-459 108 100 108 181 166 181 F F F <--- 23 460-479 108 108 108 181 180 181 F F F <--- 24 480-499 105 105 105 175 175 175 F F F 25 500-519 108 107 108 181 179 181 T T T <--- 26 520-539 96 96 96 157 157 157 T F T <--- 27 540-559 104 104 104 173 173 173 F F F 28 560-579 106 106 106 177 177 177 F F F 29 580-599 108 105 108 182 175 182 F F F <--- Max gen: 108, Max flush: 182 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 54 Verifying data now No change after verify Loop: 54 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 109 109 109 183 183 183 F F F 1 020-039 105 105 105 175 175 175 F F F 2 040-059 108 108 108 182 182 182 F F F 3 060-079 103 103 103 171 171 171 F F F 4 080-099 96 96 96 157 157 157 F F F 5 100-119 106 106 106 178 178 178 F F F 6 120-139 109 109 109 183 183 183 F F F 7 140-159 108 108 108 181 181 181 F F F 8 160-179 106 106 106 178 178 178 F F F 9 180-199 107 107 107 179 179 179 F F F 10 200-219 102 102 102 170 170 170 F F F 11 220-239 109 109 109 183 183 183 F F F 12 240-259 107 107 107 179 179 179 F F F 13 260-279 107 107 107 179 179 179 F F F 14 280-299 104 104 104 174 174 174 F F F 15 300-319 108 108 108 181 181 181 F F F 16 320-339 108 108 108 181 181 181 F F F 17 340-359 107 107 107 179 179 179 F F F 18 360-379 106 106 106 178 178 178 F F F 19 380-399 106 106 106 176 176 176 F F F 20 400-419 109 109 109 183 183 183 F F F 21 420-439 99 99 99 165 165 165 F F F 22 440-459 108 108 108 181 181 181 F F F 23 460-479 108 108 108 181 181 181 F F F 24 480-499 105 105 105 175 175 175 F F F 25 500-519 109 109 109 183 183 183 F F F 26 520-539 109 109 109 183 183 183 F F F 27 540-559 104 104 104 173 173 173 F F F 28 560-579 106 106 106 177 177 177 F F F 29 580-599 108 108 108 182 182 182 F F F Max gen: 109, Max flush: 183 Begin loop 55 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:56.035376795Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1573,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:56.043433163Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1573,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:56.043730214Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1573} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:56.043756024Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1573} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:56.043764693Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1573} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:56.043776193Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1573} {"msg":"Crucible e77b1ed3-563e-48b2-8504-bfad31f71709 has session id: b69e23f1-4c29-4b3c-9eff-20fd6ef92bde","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:56.044143432Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1573,"session_id":"b69e23f1-4c29-4b3c-9eff-20fd6ef92bde"} {"msg":"Upstairs opts: Upstairs UUID: e77b1ed3-563e-48b2-8504-bfad31f71709, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:56.044166022Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1573,"session_id":"b69e23f1-4c29-4b3c-9eff-20fd6ef92bde"} {"msg":"Crucible stats registered with UUID: e77b1ed3-563e-48b2-8504-bfad31f71709","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:56.044175871Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1573,"session_id":"b69e23f1-4c29-4b3c-9eff-20fd6ef92bde"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:45:56.044184701Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1573,"session_id":"b69e23f1-4c29-4b3c-9eff-20fd6ef92bde"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:56.044330257Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1573,"":"io task","client":"0","":"downstairs","session_id":"b69e23f1-4c29-4b3c-9eff-20fd6ef92bde"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:56.044362876Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1573,"":"io task","client":"2","":"downstairs","session_id":"b69e23f1-4c29-4b3c-9eff-20fd6ef92bde"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:56.044373926Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1573,"":"io task","client":"1","":"downstairs","session_id":"b69e23f1-4c29-4b3c-9eff-20fd6ef92bde"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:56.044495682Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1573,"task":"crutest"} {"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:56.044514382Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1573} {"msg":"e77b1ed3-563e-48b2-8504-bfad31f71709 active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:56.044537711Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1573,"session_id":"b69e23f1-4c29-4b3c-9eff-20fd6ef92bde"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:56.044547001Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1573,"client":"0","":"downstairs","session_id":"b69e23f1-4c29-4b3c-9eff-20fd6ef92bde"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:56.04455853Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1573,"client":"1","":"downstairs","session_id":"b69e23f1-4c29-4b3c-9eff-20fd6ef92bde"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:56.04456857Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1573,"client":"2","":"downstairs","session_id":"b69e23f1-4c29-4b3c-9eff-20fd6ef92bde"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:56.04457793Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1573,"":"io task","client":"2","":"downstairs","session_id":"b69e23f1-4c29-4b3c-9eff-20fd6ef92bde"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:56.04458779Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1573,"":"io task","client":"0","":"downstairs","session_id":"b69e23f1-4c29-4b3c-9eff-20fd6ef92bde"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:56.044597709Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1573,"":"io task","client":"1","":"downstairs","session_id":"b69e23f1-4c29-4b3c-9eff-20fd6ef92bde"} {"msg":"ds_connection connected from Ok(127.0.0.1:34477)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:56.044646168Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1573,"":"io task","client":"1","":"downstairs","session_id":"b69e23f1-4c29-4b3c-9eff-20fd6ef92bde"} {"msg":"ds_connection connected from Ok(127.0.0.1:48438)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:56.044670357Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1573,"":"io task","client":"0","":"downstairs","session_id":"b69e23f1-4c29-4b3c-9eff-20fd6ef92bde"} {"msg":"ds_connection connected from Ok(127.0.0.1:54849)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:56.044681437Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1573,"":"io task","client":"2","":"downstairs","session_id":"b69e23f1-4c29-4b3c-9eff-20fd6ef92bde"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:56.045336178Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1573,"client":"0","":"downstairs","session_id":"b69e23f1-4c29-4b3c-9eff-20fd6ef92bde"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:56.045376377Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1573,"client":"1","":"downstairs","session_id":"b69e23f1-4c29-4b3c-9eff-20fd6ef92bde"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:56.045390196Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1573,"client":"2","":"downstairs","session_id":"b69e23f1-4c29-4b3c-9eff-20fd6ef92bde"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:56.045680708Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1573,"session_id":"b69e23f1-4c29-4b3c-9eff-20fd6ef92bde"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:56.045690448Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1573,"session_id":"b69e23f1-4c29-4b3c-9eff-20fd6ef92bde"} {"msg":"[0]R flush_numbers[0..12]: [183, 175, 182, 171, 157, 178, 183, 181, 178, 179, 170, 183]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:56.045724127Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1573,"":"downstairs","":"downstairs","session_id":"b69e23f1-4c29-4b3c-9eff-20fd6ef92bde"} {"msg":"[0]R generation[0..12]: [109, 105, 108, 103, 96, 106, 109, 108, 106, 107, 102, 109]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:56.045735216Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1573,"":"downstairs","":"downstairs","session_id":"b69e23f1-4c29-4b3c-9eff-20fd6ef92bde"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:56.045747736Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1573,"":"downstairs","":"downstairs","session_id":"b69e23f1-4c29-4b3c-9eff-20fd6ef92bde"} {"msg":"[1]R flush_numbers[0..12]: [183, 175, 182, 171, 157, 178, 183, 181, 178, 179, 170, 183]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:56.045758196Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1573,"":"downstairs","":"downstairs","session_id":"b69e23f1-4c29-4b3c-9eff-20fd6ef92bde"} {"msg":"[1]R generation[0..12]: [109, 105, 108, 103, 96, 106, 109, 108, 106, 107, 102, 109]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:56.045768795Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1573,"":"downstairs","":"downstairs","session_id":"b69e23f1-4c29-4b3c-9eff-20fd6ef92bde"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:56.045779695Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1573,"":"downstairs","":"downstairs","session_id":"b69e23f1-4c29-4b3c-9eff-20fd6ef92bde"} {"msg":"[2]R flush_numbers[0..12]: [183, 175, 182, 171, 157, 178, 183, 181, 178, 179, 170, 183]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:56.045790315Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1573,"":"downstairs","":"downstairs","session_id":"b69e23f1-4c29-4b3c-9eff-20fd6ef92bde"} {"msg":"[2]R generation[0..12]: [109, 105, 108, 103, 96, 106, 109, 108, 106, 107, 102, 109]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:56.045800655Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1573,"":"downstairs","":"downstairs","session_id":"b69e23f1-4c29-4b3c-9eff-20fd6ef92bde"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:56.045811364Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1573,"":"downstairs","":"downstairs","session_id":"b69e23f1-4c29-4b3c-9eff-20fd6ef92bde"} {"msg":"Max found gen is 110","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:56.045820794Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1573,"":"downstairs","":"downstairs","session_id":"b69e23f1-4c29-4b3c-9eff-20fd6ef92bde"} {"msg":"Generation requested: 110 >= found:110","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:56.045830454Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1573,"":"downstairs","":"downstairs","session_id":"b69e23f1-4c29-4b3c-9eff-20fd6ef92bde"} {"msg":"Next flush: 184","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:56.045839853Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1573,"":"downstairs","":"downstairs","session_id":"b69e23f1-4c29-4b3c-9eff-20fd6ef92bde"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:56.045849233Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1573,"":"downstairs","":"downstairs","session_id":"b69e23f1-4c29-4b3c-9eff-20fd6ef92bde"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:56.045859083Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1573,"session_id":"b69e23f1-4c29-4b3c-9eff-20fd6ef92bde"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:56.045867883Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1573,"client":"0","":"downstairs","session_id":"b69e23f1-4c29-4b3c-9eff-20fd6ef92bde"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:56.045877822Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1573,"client":"1","":"downstairs","session_id":"b69e23f1-4c29-4b3c-9eff-20fd6ef92bde"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:56.045887632Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1573,"client":"2","":"downstairs","session_id":"b69e23f1-4c29-4b3c-9eff-20fd6ef92bde"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:56.045898842Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1573,"session_id":"b69e23f1-4c29-4b3c-9eff-20fd6ef92bde"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:56.045907931Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1573,"session_id":"b69e23f1-4c29-4b3c-9eff-20fd6ef92bde"} {"msg":"e77b1ed3-563e-48b2-8504-bfad31f71709 is now active with session: b69e23f1-4c29-4b3c-9eff-20fd6ef92bde","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:56.045917881Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1573,"session_id":"b69e23f1-4c29-4b3c-9eff-20fd6ef92bde"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:56.045926601Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1573,"session_id":"b69e23f1-4c29-4b3c-9eff-20fd6ef92bde"} {"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:56.045935671Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1573} {"msg":"Activated sub_volume e77b1ed3-563e-48b2-8504-bfad31f71709","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:56.04594367Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1573} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:56.04595203Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1573,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:56.046285031Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1573,"task":"crutest"} 01/30 Write block 586 len 12288 data: 10 13 12 02/30 Write block 399 len 32768 data: 9 10 10 11 10 11 10 8 03/30 Write block 172 len 12288 data: 8 9 9 04/30 Read block 170 len 28672 05/30 Read block 527 len 4096 06/30 Read block 428 len 20480 07/30 Read block 547 len 12288 08/30 Write block 58 len 32768 data: 11 9 7 10 10 7 5 6 09/30 Flush 10/30 Write block 289 len 28672 data: 8 9 9 9 10 11 11 11/30 Flush 12/30 Write block 163 len 32768 data: 9 6 9 11 10 10 10 10 13/30 Write block 382 len 4096 data: 14 14/30 Write block 227 len 28672 data: 15 12 12 12 13 8 10 15/30 Read block 358 len 12288 16/30 Write block 304 len 28672 data: 8 8 6 6 5 7 6 17/30 Read block 444 len 32768 18/30 Read block 204 len 28672 19/30 Write block 275 len 8192 data: 5 6 20/30 Read block 250 len 16384 21/30 Read block 269 len 16384 22/30 Read block 582 len 8192 23/30 Read block 424 len 24576 24/30 Read block 215 len 28672 25/30 Write block 293 len 4096 data: 11 26/30 Write block 501 len 28672 data: 11 11 13 13 10 10 11 27/30 Read block 515 len 40960 28/30 Write block 203 len 4096 data: 13 29/30 Write block 488 len 28672 data: 11 11 12 12 10 8 7 30/30 Read block 578 len 40960 ---------------------------------------------------------------- Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Sent false 1001 Acked Write 8 Done Done Sent false 1002 Acked Write 3 Done Done Sent false 1003 Acked Read 7 Done Done Sent false 1004 Acked Read 1 Done Done Sent false 1005 Acked Read 5 Done Done Sent false 1006 Acked Read 3 Done Done Sent false 1007 Acked Write 8 Done Done Sent false 1008 Acked Flush 0 Done Done Sent false 1009 Acked Write 7 Done Done Sent false 1010 Acked Flush 0 Done Done Sent false 1011 Acked Write 8 Done Done Sent false 1012 Acked Write 1 Done Done Sent false 1013 Acked Write 7 Done Done Sent false 1014 Acked Read 3 Done Done Sent false 1015 Acked Write 7 Done Done Sent false 1016 Acked Read 8 Done Done Sent false 1017 Acked Read 7 Done Done Sent false 1018 Acked Write 2 Done Done Sent false 1019 Acked Read 4 Done Done Sent false 1020 Acked Read 4 Done Done Sent false 1021 Acked Read 2 Done Done Sent false 1022 Acked Read 6 Done Done Sent false 1023 Acked Read 7 Done Done Sent false 1024 Acked Write 1 Done Done Sent false 1025 Acked Write 7 Done Done Sent false 1026 Acked Read 10 Done Done Sent false 1027 Acked Write 1 Done Done Sent false 1028 Acked Write 7 Done Done Sent false 1029 Acked Read 10 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 30 31 Done 29 30 0 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1010)) Some(JobId(1010)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:45:56.054606571Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1573} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:45:56.054782815Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1573,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 109 109 109 183 183 183 F F F 1 020-039 105 105 105 175 175 175 F F F 2 040-059 110 110 108 184 184 182 F F F <--- 3 060-079 110 110 103 184 184 171 F F F <--- 4 080-099 96 96 96 157 157 157 F F F 5 100-119 106 106 106 178 178 178 F F F 6 120-139 109 109 109 183 183 183 F F F 7 140-159 108 108 108 181 181 181 F F F 8 160-179 110 110 106 184 184 178 T T F <--- 9 180-199 107 107 107 179 179 179 F F F 10 200-219 102 102 102 170 170 170 T T F <--- 11 220-239 109 109 109 183 183 183 T T F <--- 12 240-259 107 107 107 179 179 179 F F F 13 260-279 107 107 107 179 179 179 T T F <--- 14 280-299 110 110 104 185 185 174 T T F <--- 15 300-319 108 108 108 181 181 181 T T F <--- 16 320-339 108 108 108 181 181 181 F F F 17 340-359 107 107 107 179 179 179 F F F 18 360-379 106 106 106 178 178 178 F F F 19 380-399 110 110 106 184 184 176 T T F <--- 20 400-419 110 110 109 184 184 183 F F F <--- 21 420-439 99 99 99 165 165 165 F F F 22 440-459 108 108 108 181 181 181 F F F 23 460-479 108 108 108 181 181 181 F F F 24 480-499 105 105 105 175 175 175 T T F <--- 25 500-519 109 109 109 183 183 183 T T F <--- 26 520-539 109 109 109 183 183 183 F F F 27 540-559 104 104 104 173 173 173 F F F 28 560-579 106 106 106 177 177 177 F F F 29 580-599 110 110 108 184 184 182 F F F <--- Max gen: 110, Max flush: 185 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 55 Verifying data now No change after verify Loop: 55 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 109 109 109 183 183 183 F F F 1 020-039 105 105 105 175 175 175 F F F 2 040-059 110 110 110 184 184 184 F F F 3 060-079 110 110 110 184 184 184 F F F 4 080-099 96 96 96 157 157 157 F F F 5 100-119 106 106 106 178 178 178 F F F 6 120-139 109 109 109 183 183 183 F F F 7 140-159 108 108 108 181 181 181 F F F 8 160-179 111 111 111 186 186 186 F F F 9 180-199 107 107 107 179 179 179 F F F 10 200-219 111 111 111 186 186 186 F F F 11 220-239 111 111 111 186 186 186 F F F 12 240-259 107 107 107 179 179 179 F F F 13 260-279 111 111 111 186 186 186 F F F 14 280-299 111 111 111 186 186 186 F F F 15 300-319 111 111 111 186 186 186 F F F 16 320-339 108 108 108 181 181 181 F F F 17 340-359 107 107 107 179 179 179 F F F 18 360-379 106 106 106 178 178 178 F F F 19 380-399 111 111 111 186 186 186 F F F 20 400-419 110 110 110 184 184 184 F F F 21 420-439 99 99 99 165 165 165 F F F 22 440-459 108 108 108 181 181 181 F F F 23 460-479 108 108 108 181 181 181 F F F 24 480-499 111 111 111 186 186 186 F F F 25 500-519 111 111 111 186 186 186 F F F 26 520-539 109 109 109 183 183 183 F F F 27 540-559 104 104 104 173 173 173 F F F 28 560-579 106 106 106 177 177 177 F F F 29 580-599 110 110 110 184 184 184 F F F Max gen: 111, Max flush: 186 Begin loop 56 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:00.301116852Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1581,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:00.309073412Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1581,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:00.309354974Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1581} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:00.309382653Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1581} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:00.309391253Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1581} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:00.309402643Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1581} {"msg":"Crucible 6c9c2354-e537-43e3-bfbe-539287113e64 has session id: 57eb8112-ba49-41da-8011-bbad101fa71d","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:00.309704964Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1581,"session_id":"57eb8112-ba49-41da-8011-bbad101fa71d"} {"msg":"Upstairs opts: Upstairs UUID: 6c9c2354-e537-43e3-bfbe-539287113e64, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:00.309737933Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1581,"session_id":"57eb8112-ba49-41da-8011-bbad101fa71d"} {"msg":"Crucible stats registered with UUID: 6c9c2354-e537-43e3-bfbe-539287113e64","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:00.309748673Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1581,"session_id":"57eb8112-ba49-41da-8011-bbad101fa71d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:46:00.309757742Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1581,"session_id":"57eb8112-ba49-41da-8011-bbad101fa71d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:00.309903298Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1581,"":"io task","client":"0","":"downstairs","session_id":"57eb8112-ba49-41da-8011-bbad101fa71d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:00.309935667Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1581,"":"io task","client":"1","":"downstairs","session_id":"57eb8112-ba49-41da-8011-bbad101fa71d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:00.309946447Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1581,"":"io task","client":"2","":"downstairs","session_id":"57eb8112-ba49-41da-8011-bbad101fa71d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:00.310091923Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1581,"task":"crutest"} {"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:00.310105572Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1581} {"msg":"6c9c2354-e537-43e3-bfbe-539287113e64 active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:00.310122132Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1581,"session_id":"57eb8112-ba49-41da-8011-bbad101fa71d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:00.310130892Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1581,"client":"0","":"downstairs","session_id":"57eb8112-ba49-41da-8011-bbad101fa71d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:00.310140501Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1581,"client":"1","":"downstairs","session_id":"57eb8112-ba49-41da-8011-bbad101fa71d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:00.310149961Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1581,"client":"2","":"downstairs","session_id":"57eb8112-ba49-41da-8011-bbad101fa71d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:00.310159521Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1581,"":"io task","client":"2","":"downstairs","session_id":"57eb8112-ba49-41da-8011-bbad101fa71d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:00.31016936Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1581,"":"io task","client":"0","":"downstairs","session_id":"57eb8112-ba49-41da-8011-bbad101fa71d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:00.31017947Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1581,"":"io task","client":"1","":"downstairs","session_id":"57eb8112-ba49-41da-8011-bbad101fa71d"} {"msg":"ds_connection connected from Ok(127.0.0.1:62587)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:00.31019481Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1581,"":"io task","client":"2","":"downstairs","session_id":"57eb8112-ba49-41da-8011-bbad101fa71d"} {"msg":"ds_connection connected from Ok(127.0.0.1:42500)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:00.310236569Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1581,"":"io task","client":"1","":"downstairs","session_id":"57eb8112-ba49-41da-8011-bbad101fa71d"} {"msg":"ds_connection connected from Ok(127.0.0.1:57705)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:00.310247368Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1581,"":"io task","client":"0","":"downstairs","session_id":"57eb8112-ba49-41da-8011-bbad101fa71d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:00.310900689Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1581,"client":"1","":"downstairs","session_id":"57eb8112-ba49-41da-8011-bbad101fa71d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:00.310927159Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1581,"client":"0","":"downstairs","session_id":"57eb8112-ba49-41da-8011-bbad101fa71d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:00.310961668Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1581,"client":"2","":"downstairs","session_id":"57eb8112-ba49-41da-8011-bbad101fa71d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:00.3112285Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1581,"session_id":"57eb8112-ba49-41da-8011-bbad101fa71d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:00.311264929Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1581,"session_id":"57eb8112-ba49-41da-8011-bbad101fa71d"} {"msg":"[0]R flush_numbers[0..12]: [183, 175, 184, 184, 157, 178, 183, 181, 186, 179, 186, 186]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:00.311306308Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1581,"":"downstairs","":"downstairs","session_id":"57eb8112-ba49-41da-8011-bbad101fa71d"} {"msg":"[0]R generation[0..12]: [109, 105, 110, 110, 96, 106, 109, 108, 111, 107, 111, 111]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:00.311317687Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1581,"":"downstairs","":"downstairs","session_id":"57eb8112-ba49-41da-8011-bbad101fa71d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:00.311328737Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1581,"":"downstairs","":"downstairs","session_id":"57eb8112-ba49-41da-8011-bbad101fa71d"} {"msg":"[1]R flush_numbers[0..12]: [183, 175, 184, 184, 157, 178, 183, 181, 186, 179, 186, 186]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:00.311339307Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1581,"":"downstairs","":"downstairs","session_id":"57eb8112-ba49-41da-8011-bbad101fa71d"} {"msg":"[1]R generation[0..12]: [109, 105, 110, 110, 96, 106, 109, 108, 111, 107, 111, 111]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:00.311349926Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1581,"":"downstairs","":"downstairs","session_id":"57eb8112-ba49-41da-8011-bbad101fa71d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:00.311360686Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1581,"":"downstairs","":"downstairs","session_id":"57eb8112-ba49-41da-8011-bbad101fa71d"} {"msg":"[2]R flush_numbers[0..12]: [183, 175, 184, 184, 157, 178, 183, 181, 186, 179, 186, 186]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:00.311373026Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1581,"":"downstairs","":"downstairs","session_id":"57eb8112-ba49-41da-8011-bbad101fa71d"} {"msg":"[2]R generation[0..12]: [109, 105, 110, 110, 96, 106, 109, 108, 111, 107, 111, 111]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:00.311383575Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1581,"":"downstairs","":"downstairs","session_id":"57eb8112-ba49-41da-8011-bbad101fa71d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:00.311394425Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1581,"":"downstairs","":"downstairs","session_id":"57eb8112-ba49-41da-8011-bbad101fa71d"} {"msg":"Max found gen is 112","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:00.311403815Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1581,"":"downstairs","":"downstairs","session_id":"57eb8112-ba49-41da-8011-bbad101fa71d"} {"msg":"Generation requested: 112 >= found:112","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:00.311413555Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1581,"":"downstairs","":"downstairs","session_id":"57eb8112-ba49-41da-8011-bbad101fa71d"} {"msg":"Next flush: 187","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:00.311422964Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1581,"":"downstairs","":"downstairs","session_id":"57eb8112-ba49-41da-8011-bbad101fa71d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:00.311432334Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1581,"":"downstairs","":"downstairs","session_id":"57eb8112-ba49-41da-8011-bbad101fa71d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:00.311442244Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1581,"session_id":"57eb8112-ba49-41da-8011-bbad101fa71d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:00.311450923Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1581,"client":"0","":"downstairs","session_id":"57eb8112-ba49-41da-8011-bbad101fa71d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:00.311460813Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1581,"client":"1","":"downstairs","session_id":"57eb8112-ba49-41da-8011-bbad101fa71d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:00.311470543Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1581,"client":"2","":"downstairs","session_id":"57eb8112-ba49-41da-8011-bbad101fa71d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:00.311480543Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1581,"session_id":"57eb8112-ba49-41da-8011-bbad101fa71d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:00.311489702Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1581,"session_id":"57eb8112-ba49-41da-8011-bbad101fa71d"} {"msg":"6c9c2354-e537-43e3-bfbe-539287113e64 is now active with session: 57eb8112-ba49-41da-8011-bbad101fa71d","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:00.311499512Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1581,"session_id":"57eb8112-ba49-41da-8011-bbad101fa71d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:00.311508182Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1581,"session_id":"57eb8112-ba49-41da-8011-bbad101fa71d"} {"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:00.311517122Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1581} {"msg":"Activated sub_volume 6c9c2354-e537-43e3-bfbe-539287113e64","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:00.311526301Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1581} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:00.311534241Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1581,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:00.311849402Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1581,"task":"crutest"} 01/30 Write block 50 len 8192 data: 10 8 02/30 Write block 382 len 28672 data: 15 11 10 10 8 9 9 03/30 Read block 155 len 32768 04/30 Write block 59 len 24576 data: 10 8 11 11 8 6 05/30 Read block 80 len 8192 06/30 Write block 301 len 28672 data: 10 7 8 9 9 7 7 07/30 Write block 204 len 40960 data: 15 13 12 10 12 13 13 11 12 13 08/30 Read block 201 len 36864 09/30 Write block 115 len 16384 data: 9 10 10 9 10/30 Write block 582 len 28672 data: 10 10 9 9 11 14 13 11/30 Write block 83 len 20480 data: 8 8 8 7 6 12/30 Read block 322 len 24576 13/30 Write block 101 len 32768 data: 4 4 4 4 3 3 3 3 14/30 Write block 97 len 16384 data: 9 8 8 8 15/30 Write block 207 len 24576 data: 11 13 14 14 12 13 16/30 Write block 97 len 12288 data: 10 9 9 17/30 Read block 263 len 16384 18/30 Write block 543 len 28672 data: 10 9 9 8 8 11 11 19/30 Read block 176 len 4096 20/30 Write block 194 len 40960 data: 10 9 10 10 10 9 7 13 14 14 21/30 Write block 178 len 36864 data: 9 9 9 8 9 9 12 11 12 22/30 Write block 198 len 8192 data: 11 10 23/30 Read block 540 len 16384 24/30 Write block 473 len 32768 data: 4 7 10 14 13 13 11 9 25/30 Read block 16 len 24576 26/30 Write block 376 len 12288 data: 12 12 13 27/30 Read block 185 len 4096 28/30 Write block 60 len 24576 data: 9 12 12 9 7 7 29/30 Write block 271 len 12288 data: 7 6 8 30/30 Write block 4 len 32768 data: 8 10 13 11 12 10 10 8 ---------------------------------------------------------------- Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Write 7 Done Done Done false 1002 Acked Read 8 Done Done Done false 1003 Acked Write 6 Done Done Sent false 1004 Acked Read 2 Done Done Sent false 1005 Acked Write 7 Done Done Sent false 1006 Acked Write 10 Done Done Sent false 1007 Acked Read 9 Done Done Sent false 1008 Acked Write 4 Done Done Sent false 1009 Acked Write 7 Done Done Sent false 1010 Acked Write 5 Done Done Sent false 1011 Acked Read 6 Done Done Sent false 1012 Acked Write 8 Done Done Sent false 1013 Acked Write 4 Done Done Sent false 1014 Acked Write 6 Done Done Sent false 1015 Acked Write 3 Done Done Sent false 1016 Acked Read 4 Done Done Sent false 1017 Acked Write 7 Done Done Sent false 1018 Acked Read 1 Done Done Sent false 1019 Acked Write 10 Done Done Sent false 1020 Acked Write 9 Done Done Sent false 1021 Acked Write 2 Done Done Sent false 1022 Acked Read 4 Done Done Sent false 1023 Acked Write 8 Done Done Sent false 1024 Acked Read 6 Done Done Sent false 1025 Acked Write 3 Done Done Sent false 1026 Acked Read 1 Done Done Sent false 1027 Acked Write 6 Sent Sent Sent false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 27 33 Done 27 27 3 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:46:00.31921344Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1581} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:00.319385655Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1581,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 109 109 109 183 183 183 T T F <--- 1 020-039 105 105 105 175 175 175 F F F 2 040-059 110 110 110 184 184 184 T T T 3 060-079 110 110 110 184 184 184 T T F <--- 4 080-099 96 96 96 157 157 157 T T F <--- 5 100-119 106 106 106 178 178 178 T T F <--- 6 120-139 109 109 109 183 183 183 F F F 7 140-159 108 108 108 181 181 181 F F F 8 160-179 111 111 111 186 186 186 T T F <--- 9 180-199 107 107 107 179 179 179 T T F <--- 10 200-219 111 111 111 186 186 186 T T F <--- 11 220-239 111 111 111 186 186 186 F F F 12 240-259 107 107 107 179 179 179 F F F 13 260-279 111 111 111 186 186 186 T T F <--- 14 280-299 111 111 111 186 186 186 F F F 15 300-319 111 111 111 186 186 186 T T F <--- 16 320-339 108 108 108 181 181 181 F F F 17 340-359 107 107 107 179 179 179 F F F 18 360-379 106 106 106 178 178 178 T T F <--- 19 380-399 111 111 111 186 186 186 T T T 20 400-419 110 110 110 184 184 184 F F F 21 420-439 99 99 99 165 165 165 F F F 22 440-459 108 108 108 181 181 181 F F F 23 460-479 108 108 108 181 181 181 T T F <--- 24 480-499 111 111 111 186 186 186 T T F <--- 25 500-519 111 111 111 186 186 186 F F F 26 520-539 109 109 109 183 183 183 F F F 27 540-559 104 104 104 173 173 173 T T F <--- 28 560-579 106 106 106 177 177 177 F F F 29 580-599 110 110 110 184 184 184 T T F <--- Max gen: 111, Max flush: 186 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 56 Verifying data now No change after verify Loop: 56 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 187 187 187 F F F 1 020-039 105 105 105 175 175 175 F F F 2 040-059 112 112 112 187 187 187 F F F 3 060-079 112 112 112 187 187 187 F F F 4 080-099 112 112 112 187 187 187 F F F 5 100-119 112 112 112 187 187 187 F F F 6 120-139 109 109 109 183 183 183 F F F 7 140-159 108 108 108 181 181 181 F F F 8 160-179 112 112 112 187 187 187 F F F 9 180-199 112 112 112 187 187 187 F F F 10 200-219 112 112 112 187 187 187 F F F 11 220-239 111 111 111 186 186 186 F F F 12 240-259 107 107 107 179 179 179 F F F 13 260-279 112 112 112 187 187 187 F F F 14 280-299 111 111 111 186 186 186 F F F 15 300-319 112 112 112 187 187 187 F F F 16 320-339 108 108 108 181 181 181 F F F 17 340-359 107 107 107 179 179 179 F F F 18 360-379 112 112 112 187 187 187 F F F 19 380-399 112 112 112 187 187 187 F F F 20 400-419 110 110 110 184 184 184 F F F 21 420-439 99 99 99 165 165 165 F F F 22 440-459 108 108 108 181 181 181 F F F 23 460-479 112 112 112 187 187 187 F F F 24 480-499 112 112 112 187 187 187 F F F 25 500-519 111 111 111 186 186 186 F F F 26 520-539 109 109 109 183 183 183 F F F 27 540-559 112 112 112 187 187 187 F F F 28 560-579 106 106 106 177 177 177 F F F 29 580-599 112 112 112 187 187 187 F F F Max gen: 112, Max flush: 187 Begin loop 57 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:04.579937541Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1589,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:04.587899261Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1589,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:04.588226781Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1589} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:04.58826658Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1589} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:04.58827619Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1589} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:04.58828797Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1589} {"msg":"Crucible aab54516-ed98-40a4-ba15-98bf9fce50da has session id: 357d70fd-0144-4db0-935d-5416c8804c79","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:04.588567152Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1589,"session_id":"357d70fd-0144-4db0-935d-5416c8804c79"} {"msg":"Upstairs opts: Upstairs UUID: aab54516-ed98-40a4-ba15-98bf9fce50da, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:04.5886126Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1589,"session_id":"357d70fd-0144-4db0-935d-5416c8804c79"} {"msg":"Crucible stats registered with UUID: aab54516-ed98-40a4-ba15-98bf9fce50da","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:04.5886244Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1589,"session_id":"357d70fd-0144-4db0-935d-5416c8804c79"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:46:04.58863386Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1589,"session_id":"357d70fd-0144-4db0-935d-5416c8804c79"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:04.588767586Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1589,"":"io task","client":"1","":"downstairs","session_id":"357d70fd-0144-4db0-935d-5416c8804c79"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:04.588799975Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1589,"":"io task","client":"0","":"downstairs","session_id":"357d70fd-0144-4db0-935d-5416c8804c79"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:04.588810705Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1589,"":"io task","client":"2","":"downstairs","session_id":"357d70fd-0144-4db0-935d-5416c8804c79"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:04.588937571Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1589,"task":"crutest"} {"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:04.58895544Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1589} {"msg":"aab54516-ed98-40a4-ba15-98bf9fce50da active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:04.58897912Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1589,"session_id":"357d70fd-0144-4db0-935d-5416c8804c79"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:04.58898836Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1589,"client":"0","":"downstairs","session_id":"357d70fd-0144-4db0-935d-5416c8804c79"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:04.588997969Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1589,"client":"1","":"downstairs","session_id":"357d70fd-0144-4db0-935d-5416c8804c79"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:04.589007859Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1589,"client":"2","":"downstairs","session_id":"357d70fd-0144-4db0-935d-5416c8804c79"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:04.589017429Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1589,"":"io task","client":"2","":"downstairs","session_id":"357d70fd-0144-4db0-935d-5416c8804c79"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:04.589052178Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1589,"":"io task","client":"0","":"downstairs","session_id":"357d70fd-0144-4db0-935d-5416c8804c79"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:04.589067627Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1589,"":"io task","client":"1","":"downstairs","session_id":"357d70fd-0144-4db0-935d-5416c8804c79"} {"msg":"ds_connection connected from Ok(127.0.0.1:39503)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:04.589127815Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1589,"":"io task","client":"1","":"downstairs","session_id":"357d70fd-0144-4db0-935d-5416c8804c79"} {"msg":"ds_connection connected from Ok(127.0.0.1:39153)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:04.589209173Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1589,"":"io task","client":"2","":"downstairs","session_id":"357d70fd-0144-4db0-935d-5416c8804c79"} {"msg":"ds_connection connected from Ok(127.0.0.1:55731)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:04.589225273Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1589,"":"io task","client":"0","":"downstairs","session_id":"357d70fd-0144-4db0-935d-5416c8804c79"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:04.589705609Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1589,"client":"0","":"downstairs","session_id":"357d70fd-0144-4db0-935d-5416c8804c79"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:04.589742948Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1589,"client":"1","":"downstairs","session_id":"357d70fd-0144-4db0-935d-5416c8804c79"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:04.589837965Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1589,"client":"2","":"downstairs","session_id":"357d70fd-0144-4db0-935d-5416c8804c79"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:04.59001268Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1589,"session_id":"357d70fd-0144-4db0-935d-5416c8804c79"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:04.590036289Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1589,"session_id":"357d70fd-0144-4db0-935d-5416c8804c79"} {"msg":"[0]R flush_numbers[0..12]: [187, 175, 187, 187, 187, 187, 183, 181, 187, 187, 187, 186]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:04.590128887Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1589,"":"downstairs","":"downstairs","session_id":"357d70fd-0144-4db0-935d-5416c8804c79"} {"msg":"[0]R generation[0..12]: [112, 105, 112, 112, 112, 112, 109, 108, 112, 112, 112, 111]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:04.590140556Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1589,"":"downstairs","":"downstairs","session_id":"357d70fd-0144-4db0-935d-5416c8804c79"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:04.590151566Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1589,"":"downstairs","":"downstairs","session_id":"357d70fd-0144-4db0-935d-5416c8804c79"} {"msg":"[1]R flush_numbers[0..12]: [187, 175, 187, 187, 187, 187, 183, 181, 187, 187, 187, 186]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:04.590162146Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1589,"":"downstairs","":"downstairs","session_id":"357d70fd-0144-4db0-935d-5416c8804c79"} {"msg":"[1]R generation[0..12]: [112, 105, 112, 112, 112, 112, 109, 108, 112, 112, 112, 111]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:04.590172675Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1589,"":"downstairs","":"downstairs","session_id":"357d70fd-0144-4db0-935d-5416c8804c79"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:04.590183645Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1589,"":"downstairs","":"downstairs","session_id":"357d70fd-0144-4db0-935d-5416c8804c79"} {"msg":"[2]R flush_numbers[0..12]: [187, 175, 187, 187, 187, 187, 183, 181, 187, 187, 187, 186]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:04.590194145Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1589,"":"downstairs","":"downstairs","session_id":"357d70fd-0144-4db0-935d-5416c8804c79"} {"msg":"[2]R generation[0..12]: [112, 105, 112, 112, 112, 112, 109, 108, 112, 112, 112, 111]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:04.590204664Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1589,"":"downstairs","":"downstairs","session_id":"357d70fd-0144-4db0-935d-5416c8804c79"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:04.590215464Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1589,"":"downstairs","":"downstairs","session_id":"357d70fd-0144-4db0-935d-5416c8804c79"} {"msg":"Max found gen is 113","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:04.590224874Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1589,"":"downstairs","":"downstairs","session_id":"357d70fd-0144-4db0-935d-5416c8804c79"} {"msg":"Generation requested: 114 >= found:113","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:04.590237023Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1589,"":"downstairs","":"downstairs","session_id":"357d70fd-0144-4db0-935d-5416c8804c79"} {"msg":"Next flush: 188","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:04.590246343Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1589,"":"downstairs","":"downstairs","session_id":"357d70fd-0144-4db0-935d-5416c8804c79"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:04.590255593Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1589,"":"downstairs","":"downstairs","session_id":"357d70fd-0144-4db0-935d-5416c8804c79"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:04.590265233Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1589,"session_id":"357d70fd-0144-4db0-935d-5416c8804c79"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:04.590273842Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1589,"client":"0","":"downstairs","session_id":"357d70fd-0144-4db0-935d-5416c8804c79"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:04.590283452Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1589,"client":"1","":"downstairs","session_id":"357d70fd-0144-4db0-935d-5416c8804c79"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:04.590293052Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1589,"client":"2","":"downstairs","session_id":"357d70fd-0144-4db0-935d-5416c8804c79"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:04.590302722Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1589,"session_id":"357d70fd-0144-4db0-935d-5416c8804c79"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:04.590311801Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1589,"session_id":"357d70fd-0144-4db0-935d-5416c8804c79"} {"msg":"aab54516-ed98-40a4-ba15-98bf9fce50da is now active with session: 357d70fd-0144-4db0-935d-5416c8804c79","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:04.590321661Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1589,"session_id":"357d70fd-0144-4db0-935d-5416c8804c79"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:04.590330241Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1589,"session_id":"357d70fd-0144-4db0-935d-5416c8804c79"} {"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:04.590339201Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1589} {"msg":"Activated sub_volume aab54516-ed98-40a4-ba15-98bf9fce50da","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-08-03T22:46:04.59034713Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1589} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:04.5903595Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1589,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:04.590611753Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1589,"task":"crutest"} 01/30 Write block 200 len 24576 data: 8 14 15 15 16 14 02/30 Read block 152 len 12288 03/30 Read block 434 len 36864 04/30 Flush 05/30 Write block 122 len 36864 data: 10 11 8 7 7 9 9 7 8 06/30 Read block 529 len 12288 07/30 Write block 128 len 8192 data: 10 8 08/30 Write block 440 len 8192 data: 7 6 09/30 Read block 24 len 8192 10/30 Flush 11/30 Write block 140 len 36864 data: 9 12 11 10 10 9 10 9 8 12/30 Write block 501 len 8192 data: 12 12 13/30 Flush 14/30 Write block 31 len 32768 data: 9 9 10 9 8 9 8 6 15/30 Read block 435 len 28672 16/30 Write block 259 len 20480 data: 7 7 6 6 5 17/30 Read block 318 len 8192 18/30 Read block 21 len 12288 19/30 Write block 55 len 24576 data: 9 9 9 12 11 10 20/30 Read block 214 len 24576 21/30 Write block 107 len 4096 data: 4 22/30 Read block 402 len 4096 23/30 Read block 337 len 24576 24/30 Write block 275 len 28672 data: 6 7 7 7 8 7 8 25/30 Read block 383 len 28672 26/30 Read block 195 len 20480 27/30 Write block 489 len 40960 data: 12 13 13 11 9 8 9 10 6 8 28/30 Write block 213 len 32768 data: 14 12 9 10 11 10 9 9 29/30 Write block 372 len 8192 data: 6 6 30/30 Read block 140 len 16384 ---------------------------------------------------------------- Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Read 3 Done Done Done false 1002 Acked Read 9 Done Done Sent false 1003 Acked Flush 0 Done Done Sent false 1004 Acked Write 9 Done Done Sent false 1005 Acked Read 3 Done Done Sent false 1006 Acked Write 2 Done Done Sent false 1007 Acked Write 2 Done Done Sent false 1008 Acked Read 2 Done Done Sent false 1009 Acked Flush 0 Done Done Sent false 1010 Acked Write 9 Done Done Sent false 1011 Acked Write 2 Done Done Sent false 1012 Acked Flush 0 Done Done Sent false 1013 Acked Write 8 Done Done Sent false 1014 Acked Read 7 Done Done Sent false 1015 Acked Write 5 Done Done Sent false 1016 Acked Read 2 Done Done Sent false 1017 Acked Read 3 Done Done Sent false 1018 Acked Write 6 Done Done Sent false 1019 Acked Read 6 Done Done Sent false 1020 Acked Write 1 Done Done Sent false 1021 Acked Read 1 Done Done Sent false 1022 Acked Read 6 Done Done Sent false 1023 Acked Write 7 Done Done Sent false 1024 Acked Read 7 Done Done Sent false 1025 Acked Read 5 Done Done Sent false 1026 Acked Write 10 Done Done Sent false 1027 Acked Write 8 Sent Done Sent false 1028 Acked Write 2 Sent Done Sent false 1029 Acked Read 4 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 0 28 31 Done 27 30 2 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1012)) Some(JobId(1012)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:46:04.599512966Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1589} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:04.599697851Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1589,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 187 187 187 F F F 1 020-039 105 105 105 175 175 175 T T F <--- 2 040-059 112 112 112 187 187 187 T T F <--- 3 060-079 112 112 112 187 187 187 T T F <--- 4 080-099 112 112 112 187 187 187 F F F 5 100-119 112 112 112 187 187 187 T T F <--- 6 120-139 114 114 109 189 189 183 F F F <--- 7 140-159 114 114 108 190 190 181 F F F <--- 8 160-179 112 112 112 187 187 187 F F F 9 180-199 112 112 112 187 187 187 F F F 10 200-219 114 114 112 188 188 187 T T T <--- 11 220-239 111 111 111 186 186 186 T T F <--- 12 240-259 107 107 107 179 179 179 T T F <--- 13 260-279 112 112 112 187 187 187 T T F <--- 14 280-299 111 111 111 186 186 186 T T F <--- 15 300-319 112 112 112 187 187 187 F F F 16 320-339 108 108 108 181 181 181 F F F 17 340-359 107 107 107 179 179 179 F F F 18 360-379 112 112 112 187 187 187 T T F <--- 19 380-399 112 112 112 187 187 187 F F F 20 400-419 110 110 110 184 184 184 F F F 21 420-439 99 99 99 165 165 165 F F F 22 440-459 114 114 108 189 189 181 F F F <--- 23 460-479 112 112 112 187 187 187 F F F 24 480-499 112 112 112 187 187 187 T T F <--- 25 500-519 114 114 111 190 190 186 F F F <--- 26 520-539 109 109 109 183 183 183 F F F 27 540-559 112 112 112 187 187 187 F F F 28 560-579 106 106 106 177 177 177 F F F 29 580-599 112 112 112 187 187 187 F F F Max gen: 114, Max flush: 190 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 57 Verifying data now No change after verify Loop: 57 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 187 187 187 F F F 1 020-039 115 115 115 191 191 191 F F F 2 040-059 115 115 115 191 191 191 F F F 3 060-079 115 115 115 191 191 191 F F F 4 080-099 112 112 112 187 187 187 F F F 5 100-119 115 115 115 191 191 191 F F F 6 120-139 114 114 114 189 189 189 F F F 7 140-159 114 114 114 190 190 190 F F F 8 160-179 112 112 112 187 187 187 F F F 9 180-199 112 112 112 187 187 187 F F F 10 200-219 115 115 115 191 191 191 F F F 11 220-239 115 115 115 191 191 191 F F F 12 240-259 115 115 115 191 191 191 F F F 13 260-279 115 115 115 191 191 191 F F F 14 280-299 115 115 115 191 191 191 F F F 15 300-319 112 112 112 187 187 187 F F F 16 320-339 108 108 108 181 181 181 F F F 17 340-359 107 107 107 179 179 179 F F F 18 360-379 115 115 115 191 191 191 F F F 19 380-399 112 112 112 187 187 187 F F F 20 400-419 110 110 110 184 184 184 F F F 21 420-439 99 99 99 165 165 165 F F F 22 440-459 114 114 114 189 189 189 F F F 23 460-479 112 112 112 187 187 187 F F F 24 480-499 115 115 115 191 191 191 F F F 25 500-519 114 114 114 190 190 190 F F F 26 520-539 109 109 109 183 183 183 F F F 27 540-559 112 112 112 187 187 187 F F F 28 560-579 106 106 106 177 177 177 F F F 29 580-599 112 112 112 187 187 187 F F F Max gen: 115, Max flush: 191 Begin loop 58 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:08.850826588Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1597,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:46:08.857832766Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1597,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:08.857886814Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1597,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:08.864648439Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1597,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:08.864913112Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1597} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:08.864938001Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1597} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:08.864946491Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1597} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:08.86495783Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1597} {"msg":"Crucible 50ca837a-cec8-4258-8437-33b39f4ed558 has session id: 77db9e5f-07aa-474d-b257-ea0b1b3e1ae8","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:08.86532742Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1597,"session_id":"77db9e5f-07aa-474d-b257-ea0b1b3e1ae8"} {"msg":"Upstairs opts: Upstairs UUID: 50ca837a-cec8-4258-8437-33b39f4ed558, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:08.865356949Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1597,"session_id":"77db9e5f-07aa-474d-b257-ea0b1b3e1ae8"} {"msg":"Crucible stats registered with UUID: 50ca837a-cec8-4258-8437-33b39f4ed558","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:08.865367958Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1597,"session_id":"77db9e5f-07aa-474d-b257-ea0b1b3e1ae8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:46:08.865377208Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1597,"session_id":"77db9e5f-07aa-474d-b257-ea0b1b3e1ae8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:08.865509944Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1597,"":"io task","client":"2","":"downstairs","session_id":"77db9e5f-07aa-474d-b257-ea0b1b3e1ae8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:08.865543803Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1597,"":"io task","client":"0","":"downstairs","session_id":"77db9e5f-07aa-474d-b257-ea0b1b3e1ae8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:08.865554593Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1597,"":"io task","client":"1","":"downstairs","session_id":"77db9e5f-07aa-474d-b257-ea0b1b3e1ae8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:08.8656532Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1597,"task":"crutest"} {"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:08.86567131Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1597} {"msg":"50ca837a-cec8-4258-8437-33b39f4ed558 active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:08.865707879Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1597,"session_id":"77db9e5f-07aa-474d-b257-ea0b1b3e1ae8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:08.865733988Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1597,"client":"0","":"downstairs","session_id":"77db9e5f-07aa-474d-b257-ea0b1b3e1ae8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:08.865754617Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1597,"client":"1","":"downstairs","session_id":"77db9e5f-07aa-474d-b257-ea0b1b3e1ae8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:08.865764957Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1597,"client":"2","":"downstairs","session_id":"77db9e5f-07aa-474d-b257-ea0b1b3e1ae8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:08.865774427Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1597,"":"io task","client":"2","":"downstairs","session_id":"77db9e5f-07aa-474d-b257-ea0b1b3e1ae8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:08.865784446Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1597,"":"io task","client":"0","":"downstairs","session_id":"77db9e5f-07aa-474d-b257-ea0b1b3e1ae8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:08.865794406Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1597,"":"io task","client":"1","":"downstairs","session_id":"77db9e5f-07aa-474d-b257-ea0b1b3e1ae8"} {"msg":"ds_connection connected from Ok(127.0.0.1:37967)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:08.865804616Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1597,"":"io task","client":"2","":"downstairs","session_id":"77db9e5f-07aa-474d-b257-ea0b1b3e1ae8"} {"msg":"ds_connection connected from Ok(127.0.0.1:45825)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:08.865828825Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1597,"":"io task","client":"1","":"downstairs","session_id":"77db9e5f-07aa-474d-b257-ea0b1b3e1ae8"} {"msg":"ds_connection connected from Ok(127.0.0.1:46945)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:08.865840635Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1597,"":"io task","client":"0","":"downstairs","session_id":"77db9e5f-07aa-474d-b257-ea0b1b3e1ae8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:08.866482456Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1597,"client":"1","":"downstairs","session_id":"77db9e5f-07aa-474d-b257-ea0b1b3e1ae8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:08.866508436Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1597,"client":"0","":"downstairs","session_id":"77db9e5f-07aa-474d-b257-ea0b1b3e1ae8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:08.866519515Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1597,"client":"2","":"downstairs","session_id":"77db9e5f-07aa-474d-b257-ea0b1b3e1ae8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:08.866811547Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1597,"session_id":"77db9e5f-07aa-474d-b257-ea0b1b3e1ae8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:08.866850346Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1597,"session_id":"77db9e5f-07aa-474d-b257-ea0b1b3e1ae8"} {"msg":"[0]R flush_numbers[0..12]: [187, 191, 191, 191, 187, 191, 189, 190, 187, 187, 191, 191]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:08.866912924Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1597,"":"downstairs","":"downstairs","session_id":"77db9e5f-07aa-474d-b257-ea0b1b3e1ae8"} {"msg":"[0]R generation[0..12]: [112, 115, 115, 115, 112, 115, 114, 114, 112, 112, 115, 115]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:08.866924174Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1597,"":"downstairs","":"downstairs","session_id":"77db9e5f-07aa-474d-b257-ea0b1b3e1ae8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:08.866934993Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1597,"":"downstairs","":"downstairs","session_id":"77db9e5f-07aa-474d-b257-ea0b1b3e1ae8"} {"msg":"[1]R flush_numbers[0..12]: [187, 191, 191, 191, 187, 191, 189, 190, 187, 187, 191, 191]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:08.866945553Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1597,"":"downstairs","":"downstairs","session_id":"77db9e5f-07aa-474d-b257-ea0b1b3e1ae8"} {"msg":"[1]R generation[0..12]: [112, 115, 115, 115, 112, 115, 114, 114, 112, 112, 115, 115]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:08.866956003Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1597,"":"downstairs","":"downstairs","session_id":"77db9e5f-07aa-474d-b257-ea0b1b3e1ae8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:08.866966752Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1597,"":"downstairs","":"downstairs","session_id":"77db9e5f-07aa-474d-b257-ea0b1b3e1ae8"} {"msg":"[2]R flush_numbers[0..12]: [187, 191, 191, 191, 187, 191, 189, 190, 187, 187, 191, 191]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:08.866977352Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1597,"":"downstairs","":"downstairs","session_id":"77db9e5f-07aa-474d-b257-ea0b1b3e1ae8"} {"msg":"[2]R generation[0..12]: [112, 115, 115, 115, 112, 115, 114, 114, 112, 112, 115, 115]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:08.866987852Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1597,"":"downstairs","":"downstairs","session_id":"77db9e5f-07aa-474d-b257-ea0b1b3e1ae8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:08.866998631Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1597,"":"downstairs","":"downstairs","session_id":"77db9e5f-07aa-474d-b257-ea0b1b3e1ae8"} {"msg":"Max found gen is 116","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:08.867008141Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1597,"":"downstairs","":"downstairs","session_id":"77db9e5f-07aa-474d-b257-ea0b1b3e1ae8"} {"msg":"Generation requested: 116 >= found:116","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:08.867017901Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1597,"":"downstairs","":"downstairs","session_id":"77db9e5f-07aa-474d-b257-ea0b1b3e1ae8"} {"msg":"Next flush: 192","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:08.867027201Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1597,"":"downstairs","":"downstairs","session_id":"77db9e5f-07aa-474d-b257-ea0b1b3e1ae8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:08.86703649Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1597,"":"downstairs","":"downstairs","session_id":"77db9e5f-07aa-474d-b257-ea0b1b3e1ae8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:08.86704634Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1597,"session_id":"77db9e5f-07aa-474d-b257-ea0b1b3e1ae8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:08.8670567Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1597,"client":"0","":"downstairs","session_id":"77db9e5f-07aa-474d-b257-ea0b1b3e1ae8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:08.867066339Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1597,"client":"1","":"downstairs","session_id":"77db9e5f-07aa-474d-b257-ea0b1b3e1ae8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:08.867075939Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1597,"client":"2","":"downstairs","session_id":"77db9e5f-07aa-474d-b257-ea0b1b3e1ae8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:08.867085599Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1597,"session_id":"77db9e5f-07aa-474d-b257-ea0b1b3e1ae8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:08.867094539Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1597,"session_id":"77db9e5f-07aa-474d-b257-ea0b1b3e1ae8"} {"msg":"50ca837a-cec8-4258-8437-33b39f4ed558 is now active with session: 77db9e5f-07aa-474d-b257-ea0b1b3e1ae8","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:08.867104348Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1597,"session_id":"77db9e5f-07aa-474d-b257-ea0b1b3e1ae8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:08.867113108Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1597,"session_id":"77db9e5f-07aa-474d-b257-ea0b1b3e1ae8"} {"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:08.867122128Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1597} {"msg":"Activated sub_volume 50ca837a-cec8-4258-8437-33b39f4ed558","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:08.867129968Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1597} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:08.867137977Z","hostname":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1597,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:08.86740651Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1597,"task":"crutest"} 01/30 Write block 358 len 24576 data: 8 7 8 8 11 12 02/30 Flush 03/30 Write block 420 len 40960 data: 10 10 9 9 10 9 9 9 8 10 04/30 Read block 401 len 12288 05/30 Write block 272 len 36864 data: 7 9 6 7 8 8 8 9 8 06/30 Read block 49 len 32768 07/30 Read block 136 len 8192 08/30 Read block 450 len 40960 09/30 Write block 263 len 36864 data: 6 3 4 4 5 5 5 5 8 10/30 Read block 370 len 8192 11/30 Flush 12/30 Write block 195 len 20480 data: 10 11 11 12 11 13/30 Write block 0 len 12288 data: 4 4 7 14/30 Read block 158 len 8192 15/30 Write block 587 len 28672 data: 15 14 8 8 7 6 6 16/30 Flush 17/30 Write block 223 len 40960 data: 13 14 16 15 16 13 13 13 14 9 18/30 Read block 18 len 16384 19/30 Read block 331 len 16384 20/30 Read block 566 len 20480 21/30 Read block 449 len 40960 22/30 Read block 437 len 36864 23/30 Read block 173 len 8192 24/30 Write block 69 len 28672 data: 6 9 9 9 10 10 10 25/30 Write block 343 len 24576 data: 9 9 9 10 9 8 26/30 Read block 401 len 24576 27/30 Read block 24 len 20480 28/30 Write block 261 len 28672 data: 7 7 7 4 5 5 6 29/30 Read block 166 len 24576 30/30 Read block 106 len 4096 ---------------------------------------------------------------- Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Sent Done Done false 1001 Acked Flush 0 Sent Done Done false 1002 Acked Write 10 Sent Done Done false 1003 Acked Read 3 Sent Done Done false 1004 Acked Write 9 Sent Done Done false 1005 Acked Read 8 Sent Done Done false 1006 Acked Read 2 Sent Done Done false 1007 Acked Read 10 Sent Done Done false 1008 Acked Write 9 Sent Done Done false 1009 Acked Read 2 Sent Done Done false 1010 Acked Flush 0 Sent Done Done false 1011 Acked Write 5 Sent Done Done false 1012 Acked Write 3 Sent Done Done false 1013 Acked Read 2 Sent Done Done false 1014 Acked Write 7 Sent Done Done false 1015 Acked Flush 0 Sent Done Done false 1016 Acked Write 10 Sent Done Done false 1017 Acked Read 4 Sent Done Done false 1018 Acked Read 4 Sent Done Done false 1019 Acked Read 5 Sent Done Done false 1020 Acked Read 10 Sent Done Done false 1021 Acked Read 9 Sent Done Done false 1022 Acked Read 2 Sent Done Done false 1023 Acked Write 7 Sent Done Done false 1024 Acked Write 6 Sent Done Done false 1025 Acked Read 6 Sent Done Done false 1026 Acked Read 5 Sent Done Done false 1027 Acked Write 7 Sent Done Done false 1028 Acked Read 6 Sent Done Done false 1029 Acked Read 1 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 0 1 31 Done 0 30 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1015)) Some(JobId(1015)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:46:08.87745141Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1597} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:08.877625325Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1597,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 116 116 187 194 194 F F F <--- 1 020-039 115 115 115 191 191 191 F F F 2 040-059 115 115 115 191 191 191 F F F 3 060-079 115 115 115 191 191 191 F T T <--- 4 080-099 112 112 112 187 187 187 F F F 5 100-119 115 115 115 191 191 191 F F F 6 120-139 114 114 114 189 189 189 F F F 7 140-159 114 114 114 190 190 190 F F F 8 160-179 112 112 112 187 187 187 F F F 9 180-199 112 116 116 187 194 194 F F F <--- 10 200-219 115 115 115 191 191 191 F F F 11 220-239 115 115 115 191 191 191 F T T <--- 12 240-259 115 115 115 191 191 191 F F F 13 260-279 115 116 116 191 193 193 F T T <--- 14 280-299 115 116 116 191 193 193 F F F <--- 15 300-319 112 112 112 187 187 187 F F F 16 320-339 108 108 108 181 181 181 F F F 17 340-359 107 116 116 179 192 192 F T T <--- 18 360-379 115 116 116 191 192 192 F F F <--- 19 380-399 112 112 112 187 187 187 F F F 20 400-419 110 110 110 184 184 184 F F F 21 420-439 99 116 116 165 193 193 F F F <--- 22 440-459 114 114 114 189 189 189 F F F 23 460-479 112 112 112 187 187 187 F F F 24 480-499 115 115 115 191 191 191 F F F 25 500-519 114 114 114 190 190 190 F F F 26 520-539 109 109 109 183 183 183 F F F 27 540-559 112 112 112 187 187 187 F F F 28 560-579 106 106 106 177 177 177 F F F 29 580-599 112 116 116 187 194 194 F F F <--- Max gen: 116, Max flush: 194 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 58 Verifying data now No change after verify Loop: 58 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 116 116 116 194 194 194 F F F 1 020-039 115 115 115 191 191 191 F F F 2 040-059 115 115 115 191 191 191 F F F 3 060-079 117 117 117 195 195 195 F F F 4 080-099 112 112 112 187 187 187 F F F 5 100-119 115 115 115 191 191 191 F F F 6 120-139 114 114 114 189 189 189 F F F 7 140-159 114 114 114 190 190 190 F F F 8 160-179 112 112 112 187 187 187 F F F 9 180-199 116 116 116 194 194 194 F F F 10 200-219 115 115 115 191 191 191 F F F 11 220-239 117 117 117 195 195 195 F F F 12 240-259 115 115 115 191 191 191 F F F 13 260-279 117 117 117 195 195 195 F F F 14 280-299 116 116 116 193 193 193 F F F 15 300-319 112 112 112 187 187 187 F F F 16 320-339 108 108 108 181 181 181 F F F 17 340-359 117 117 117 195 195 195 F F F 18 360-379 116 116 116 192 192 192 F F F 19 380-399 112 112 112 187 187 187 F F F 20 400-419 110 110 110 184 184 184 F F F 21 420-439 116 116 116 193 193 193 F F F 22 440-459 114 114 114 189 189 189 F F F 23 460-479 112 112 112 187 187 187 F F F 24 480-499 115 115 115 191 191 191 F F F 25 500-519 114 114 114 190 190 190 F F F 26 520-539 109 109 109 183 183 183 F F F 27 540-559 112 112 112 187 187 187 F F F 28 560-579 106 106 106 177 177 177 F F F 29 580-599 116 116 116 194 194 194 F F F Max gen: 117, Max flush: 195 Begin loop 59 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:13.121679067Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1605,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:46:13.128635706Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1605,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:13.128689435Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1605,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:13.135374072Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1605,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:13.135639084Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1605} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:13.135665293Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1605} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:13.135676423Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1605} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:13.135687943Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1605} {"msg":"Crucible 6f2ea552-c2d3-4663-9089-62a1854279ea has session id: d5bc66ee-76a4-4ef6-9238-72ac2a64d634","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:13.136019413Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1605,"session_id":"d5bc66ee-76a4-4ef6-9238-72ac2a64d634"} {"msg":"Upstairs opts: Upstairs UUID: 6f2ea552-c2d3-4663-9089-62a1854279ea, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:13.136050612Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1605,"session_id":"d5bc66ee-76a4-4ef6-9238-72ac2a64d634"} {"msg":"Crucible stats registered with UUID: 6f2ea552-c2d3-4663-9089-62a1854279ea","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:13.136061712Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1605,"session_id":"d5bc66ee-76a4-4ef6-9238-72ac2a64d634"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:46:13.136070802Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1605,"session_id":"d5bc66ee-76a4-4ef6-9238-72ac2a64d634"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:13.136193608Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1605,"":"io task","client":"2","":"downstairs","session_id":"d5bc66ee-76a4-4ef6-9238-72ac2a64d634"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:13.136228577Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1605,"":"io task","client":"0","":"downstairs","session_id":"d5bc66ee-76a4-4ef6-9238-72ac2a64d634"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:13.136239417Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1605,"":"io task","client":"1","":"downstairs","session_id":"d5bc66ee-76a4-4ef6-9238-72ac2a64d634"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:13.136363443Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1605,"task":"crutest"} {"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:13.136377603Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1605} {"msg":"6f2ea552-c2d3-4663-9089-62a1854279ea active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:13.136410152Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1605,"session_id":"d5bc66ee-76a4-4ef6-9238-72ac2a64d634"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:13.136419572Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1605,"client":"0","":"downstairs","session_id":"d5bc66ee-76a4-4ef6-9238-72ac2a64d634"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:13.136429191Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1605,"client":"1","":"downstairs","session_id":"d5bc66ee-76a4-4ef6-9238-72ac2a64d634"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:13.136441121Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1605,"client":"2","":"downstairs","session_id":"d5bc66ee-76a4-4ef6-9238-72ac2a64d634"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:13.136450711Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1605,"":"io task","client":"2","":"downstairs","session_id":"d5bc66ee-76a4-4ef6-9238-72ac2a64d634"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:13.13646062Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1605,"":"io task","client":"0","":"downstairs","session_id":"d5bc66ee-76a4-4ef6-9238-72ac2a64d634"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:13.13647071Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1605,"":"io task","client":"1","":"downstairs","session_id":"d5bc66ee-76a4-4ef6-9238-72ac2a64d634"} {"msg":"ds_connection connected from Ok(127.0.0.1:61786)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:13.136543558Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1605,"":"io task","client":"1","":"downstairs","session_id":"d5bc66ee-76a4-4ef6-9238-72ac2a64d634"} {"msg":"ds_connection connected from Ok(127.0.0.1:61187)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:13.136566077Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1605,"":"io task","client":"2","":"downstairs","session_id":"d5bc66ee-76a4-4ef6-9238-72ac2a64d634"} {"msg":"ds_connection connected from Ok(127.0.0.1:63060)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:13.136580407Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1605,"":"io task","client":"0","":"downstairs","session_id":"d5bc66ee-76a4-4ef6-9238-72ac2a64d634"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:13.137247528Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1605,"client":"1","":"downstairs","session_id":"d5bc66ee-76a4-4ef6-9238-72ac2a64d634"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:13.137268387Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1605,"client":"2","":"downstairs","session_id":"d5bc66ee-76a4-4ef6-9238-72ac2a64d634"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:13.137279417Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1605,"client":"0","":"downstairs","session_id":"d5bc66ee-76a4-4ef6-9238-72ac2a64d634"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:13.137546759Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1605,"session_id":"d5bc66ee-76a4-4ef6-9238-72ac2a64d634"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:13.137556549Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1605,"session_id":"d5bc66ee-76a4-4ef6-9238-72ac2a64d634"} {"msg":"[0]R flush_numbers[0..12]: [194, 191, 191, 195, 187, 191, 189, 190, 187, 194, 191, 195]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:13.137589578Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1605,"":"downstairs","":"downstairs","session_id":"d5bc66ee-76a4-4ef6-9238-72ac2a64d634"} {"msg":"[0]R generation[0..12]: [116, 115, 115, 117, 112, 115, 114, 114, 112, 116, 115, 117]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:13.137600577Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1605,"":"downstairs","":"downstairs","session_id":"d5bc66ee-76a4-4ef6-9238-72ac2a64d634"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:13.137611637Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1605,"":"downstairs","":"downstairs","session_id":"d5bc66ee-76a4-4ef6-9238-72ac2a64d634"} {"msg":"[1]R flush_numbers[0..12]: [194, 191, 191, 195, 187, 191, 189, 190, 187, 194, 191, 195]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:13.137624447Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1605,"":"downstairs","":"downstairs","session_id":"d5bc66ee-76a4-4ef6-9238-72ac2a64d634"} {"msg":"[1]R generation[0..12]: [116, 115, 115, 117, 112, 115, 114, 114, 112, 116, 115, 117]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:13.137635146Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1605,"":"downstairs","":"downstairs","session_id":"d5bc66ee-76a4-4ef6-9238-72ac2a64d634"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:13.137645986Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1605,"":"downstairs","":"downstairs","session_id":"d5bc66ee-76a4-4ef6-9238-72ac2a64d634"} {"msg":"[2]R flush_numbers[0..12]: [194, 191, 191, 195, 187, 191, 189, 190, 187, 194, 191, 195]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:13.137656516Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1605,"":"downstairs","":"downstairs","session_id":"d5bc66ee-76a4-4ef6-9238-72ac2a64d634"} {"msg":"[2]R generation[0..12]: [116, 115, 115, 117, 112, 115, 114, 114, 112, 116, 115, 117]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:13.137666856Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1605,"":"downstairs","":"downstairs","session_id":"d5bc66ee-76a4-4ef6-9238-72ac2a64d634"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:13.137677645Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1605,"":"downstairs","":"downstairs","session_id":"d5bc66ee-76a4-4ef6-9238-72ac2a64d634"} {"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:13.137686965Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1605,"":"downstairs","":"downstairs","session_id":"d5bc66ee-76a4-4ef6-9238-72ac2a64d634"} {"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:13.137696735Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1605,"":"downstairs","":"downstairs","session_id":"d5bc66ee-76a4-4ef6-9238-72ac2a64d634"} {"msg":"Next flush: 196","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:13.137706004Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1605,"":"downstairs","":"downstairs","session_id":"d5bc66ee-76a4-4ef6-9238-72ac2a64d634"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:13.137715194Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1605,"":"downstairs","":"downstairs","session_id":"d5bc66ee-76a4-4ef6-9238-72ac2a64d634"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:13.137724794Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1605,"session_id":"d5bc66ee-76a4-4ef6-9238-72ac2a64d634"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:13.137733364Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1605,"client":"0","":"downstairs","session_id":"d5bc66ee-76a4-4ef6-9238-72ac2a64d634"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:13.137742823Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1605,"client":"1","":"downstairs","session_id":"d5bc66ee-76a4-4ef6-9238-72ac2a64d634"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:13.137752323Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1605,"client":"2","":"downstairs","session_id":"d5bc66ee-76a4-4ef6-9238-72ac2a64d634"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:13.137761953Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1605,"session_id":"d5bc66ee-76a4-4ef6-9238-72ac2a64d634"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:13.137772223Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1605,"session_id":"d5bc66ee-76a4-4ef6-9238-72ac2a64d634"} {"msg":"6f2ea552-c2d3-4663-9089-62a1854279ea is now active with session: d5bc66ee-76a4-4ef6-9238-72ac2a64d634","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:13.137782042Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1605,"session_id":"d5bc66ee-76a4-4ef6-9238-72ac2a64d634"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:13.137790692Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1605,"session_id":"d5bc66ee-76a4-4ef6-9238-72ac2a64d634"} {"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:13.137799752Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1605} {"msg":"Activated sub_volume 6f2ea552-c2d3-4663-9089-62a1854279ea","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:13.137807611Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1605} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:13.137815611Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1605,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:13.138133292Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1605,"task":"crutest"} 01/30 Write block 172 len 8192 data: 9 10 02/30 Write block 280 len 40960 data: 9 9 6 6 7 9 7 7 7 9 03/30 Write block 260 len 20480 data: 8 8 8 8 5 04/30 Read block 276 len 4096 05/30 Read block 287 len 28672 06/30 Write block 296 len 32768 data: 11 12 9 9 10 11 8 9 07/30 Write block 439 len 24576 data: 6 8 7 6 5 6 08/30 Read block 369 len 16384 09/30 Read block 396 len 32768 10/30 Read block 302 len 28672 11/30 Read block 532 len 32768 12/30 Write block 405 len 12288 data: 11 9 9 13/30 Write block 283 len 12288 data: 7 8 10 14/30 Write block 318 len 24576 data: 3 5 4 7 7 11 15/30 Read block 347 len 8192 16/30 Read block 299 len 4096 17/30 Write block 262 len 40960 data: 9 9 6 6 6 7 6 6 6 9 18/30 Read block 149 len 36864 19/30 Write block 122 len 40960 data: 11 12 9 8 8 10 11 9 9 9 20/30 Write block 584 len 16384 data: 10 10 12 16 21/30 Write block 264 len 20480 data: 7 7 7 8 7 22/30 Read block 547 len 8192 23/30 Read block 335 len 4096 24/30 Read block 370 len 16384 25/30 Write block 11 len 12288 data: 9 7 8 26/30 Read block 191 len 8192 27/30 Write block 537 len 32768 data: 15 16 15 14 11 10 11 10 28/30 Write block 4 len 4096 data: 9 29/30 Write block 402 len 20480 data: 12 11 12 12 10 30/30 Read block 74 len 24576 ---------------------------------------------------------------- Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Write 10 Sent Done Done false 1002 Acked Write 5 Sent Done Done false 1003 Acked Read 1 Sent Done Done false 1004 Acked Read 7 Sent Done Done false 1005 Acked Write 8 Sent Done Done false 1006 Acked Write 6 Sent Done Done false 1007 Acked Read 4 Sent Done Done false 1008 Acked Read 8 Sent Done Done false 1009 Acked Read 7 Sent Done Done false 1010 Acked Read 8 Sent Done Done false 1011 Acked Write 3 Sent Done Done false 1012 Acked Write 3 Sent Done Done false 1013 Acked Write 6 Sent Done Done false 1014 Acked Read 2 Sent Done Done false 1015 Acked Read 1 Sent Done Done false 1016 Acked Write 10 Sent Done Done false 1017 Acked Read 9 Sent Done Done false 1018 Acked Write 10 Sent Done Done false 1019 Acked Write 4 Sent Done Done false 1020 Acked Write 5 Sent Done Done false 1021 Acked Read 2 Sent Done Done false 1022 Acked Read 1 Sent Done Done false 1023 Acked Read 4 Sent Done Done false 1024 Acked Write 3 Sent Done Done false 1025 Acked Read 2 Sent Done Done false 1026 Acked Write 8 Sent Done Done false 1027 Acked Write 1 Sent Done Done false 1028 Acked Write 5 Sent Done Done false 1029 Acked Read 6 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 1 0 30 Done 1 29 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:46:13.145963756Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1605} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:13.146142141Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1605,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 116 116 116 194 194 194 F T T <--- 1 020-039 115 115 115 191 191 191 F F F 2 040-059 115 115 115 191 191 191 F F F 3 060-079 117 117 117 195 195 195 F F F 4 080-099 112 112 112 187 187 187 F F F 5 100-119 115 115 115 191 191 191 F F F 6 120-139 114 114 114 189 189 189 F T T <--- 7 140-159 114 114 114 190 190 190 F F F 8 160-179 112 112 112 187 187 187 T T T 9 180-199 116 116 116 194 194 194 F F F 10 200-219 115 115 115 191 191 191 F F F 11 220-239 117 117 117 195 195 195 F F F 12 240-259 115 115 115 191 191 191 F F F 13 260-279 117 117 117 195 195 195 F T T <--- 14 280-299 116 116 116 193 193 193 F T T <--- 15 300-319 112 112 112 187 187 187 F T T <--- 16 320-339 108 108 108 181 181 181 F T T <--- 17 340-359 117 117 117 195 195 195 F F F 18 360-379 116 116 116 192 192 192 F F F 19 380-399 112 112 112 187 187 187 F F F 20 400-419 110 110 110 184 184 184 F T T <--- 21 420-439 116 116 116 193 193 193 F T T <--- 22 440-459 114 114 114 189 189 189 F T T <--- 23 460-479 112 112 112 187 187 187 F F F 24 480-499 115 115 115 191 191 191 F F F 25 500-519 114 114 114 190 190 190 F F F 26 520-539 109 109 109 183 183 183 F T T <--- 27 540-559 112 112 112 187 187 187 F T T <--- 28 560-579 106 106 106 177 177 177 F F F 29 580-599 116 116 116 194 194 194 F T T <--- Max gen: 117, Max flush: 195 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 59 Verifying data now No change after verify Loop: 59 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 196 196 196 F F F 1 020-039 115 115 115 191 191 191 F F F 2 040-059 115 115 115 191 191 191 F F F 3 060-079 117 117 117 195 195 195 F F F 4 080-099 112 112 112 187 187 187 F F F 5 100-119 115 115 115 191 191 191 F F F 6 120-139 118 118 118 196 196 196 F F F 7 140-159 114 114 114 190 190 190 F F F 8 160-179 118 118 118 196 196 196 F F F 9 180-199 116 116 116 194 194 194 F F F 10 200-219 115 115 115 191 191 191 F F F 11 220-239 117 117 117 195 195 195 F F F 12 240-259 115 115 115 191 191 191 F F F 13 260-279 118 118 118 196 196 196 F F F 14 280-299 118 118 118 196 196 196 F F F 15 300-319 118 118 118 196 196 196 F F F 16 320-339 118 118 118 196 196 196 F F F 17 340-359 117 117 117 195 195 195 F F F 18 360-379 116 116 116 192 192 192 F F F 19 380-399 112 112 112 187 187 187 F F F 20 400-419 118 118 118 196 196 196 F F F 21 420-439 118 118 118 196 196 196 F F F 22 440-459 118 118 118 196 196 196 F F F 23 460-479 112 112 112 187 187 187 F F F 24 480-499 115 115 115 191 191 191 F F F 25 500-519 114 114 114 190 190 190 F F F 26 520-539 118 118 118 196 196 196 F F F 27 540-559 118 118 118 196 196 196 F F F 28 560-579 106 106 106 177 177 177 F F F 29 580-599 118 118 118 196 196 196 F F F Max gen: 118, Max flush: 196 Begin loop 60 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:17.403578217Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1613,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:17.411604635Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1613,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:17.411895927Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1613} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:17.411920226Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1613} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:17.411928586Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1613} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:17.411940066Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1613} {"msg":"Crucible b668c33e-b528-4f7b-aa35-21520cb418ca has session id: cf2150fa-3d50-4f41-9725-73610e60f044","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:17.412304835Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1613,"session_id":"cf2150fa-3d50-4f41-9725-73610e60f044"} {"msg":"Upstairs opts: Upstairs UUID: b668c33e-b528-4f7b-aa35-21520cb418ca, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:17.412334874Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1613,"session_id":"cf2150fa-3d50-4f41-9725-73610e60f044"} {"msg":"Crucible stats registered with UUID: b668c33e-b528-4f7b-aa35-21520cb418ca","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:17.412346014Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1613,"session_id":"cf2150fa-3d50-4f41-9725-73610e60f044"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:46:17.412355284Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1613,"session_id":"cf2150fa-3d50-4f41-9725-73610e60f044"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:17.41249573Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1613,"":"io task","client":"2","":"downstairs","session_id":"cf2150fa-3d50-4f41-9725-73610e60f044"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:17.412526349Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1613,"":"io task","client":"0","":"downstairs","session_id":"cf2150fa-3d50-4f41-9725-73610e60f044"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:17.412537268Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1613,"":"io task","client":"1","":"downstairs","session_id":"cf2150fa-3d50-4f41-9725-73610e60f044"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:17.412689244Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1613,"task":"crutest"} {"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:17.412703534Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1613} {"msg":"b668c33e-b528-4f7b-aa35-21520cb418ca active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:17.412721843Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1613,"session_id":"cf2150fa-3d50-4f41-9725-73610e60f044"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:17.412730593Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1613,"client":"0","":"downstairs","session_id":"cf2150fa-3d50-4f41-9725-73610e60f044"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:17.412740233Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1613,"client":"1","":"downstairs","session_id":"cf2150fa-3d50-4f41-9725-73610e60f044"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:17.412749782Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1613,"client":"2","":"downstairs","session_id":"cf2150fa-3d50-4f41-9725-73610e60f044"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:17.412759172Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1613,"":"io task","client":"2","":"downstairs","session_id":"cf2150fa-3d50-4f41-9725-73610e60f044"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:17.412768992Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1613,"":"io task","client":"0","":"downstairs","session_id":"cf2150fa-3d50-4f41-9725-73610e60f044"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:17.412778972Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1613,"":"io task","client":"1","":"downstairs","session_id":"cf2150fa-3d50-4f41-9725-73610e60f044"} {"msg":"ds_connection connected from Ok(127.0.0.1:37165)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:17.412861209Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1613,"":"io task","client":"1","":"downstairs","session_id":"cf2150fa-3d50-4f41-9725-73610e60f044"} {"msg":"ds_connection connected from Ok(127.0.0.1:43581)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:17.412896348Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1613,"":"io task","client":"0","":"downstairs","session_id":"cf2150fa-3d50-4f41-9725-73610e60f044"} {"msg":"ds_connection connected from Ok(127.0.0.1:41480)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:17.413024984Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1613,"":"io task","client":"2","":"downstairs","session_id":"cf2150fa-3d50-4f41-9725-73610e60f044"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:17.413568789Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1613,"client":"1","":"downstairs","session_id":"cf2150fa-3d50-4f41-9725-73610e60f044"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:17.413604828Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1613,"client":"2","":"downstairs","session_id":"cf2150fa-3d50-4f41-9725-73610e60f044"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:17.413620257Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1613,"client":"0","":"downstairs","session_id":"cf2150fa-3d50-4f41-9725-73610e60f044"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:17.4138833Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1613,"session_id":"cf2150fa-3d50-4f41-9725-73610e60f044"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:17.413902389Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1613,"session_id":"cf2150fa-3d50-4f41-9725-73610e60f044"} {"msg":"[0]R flush_numbers[0..12]: [196, 191, 191, 195, 187, 191, 196, 190, 196, 194, 191, 195]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:17.413920459Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1613,"":"downstairs","":"downstairs","session_id":"cf2150fa-3d50-4f41-9725-73610e60f044"} {"msg":"[0]R generation[0..12]: [118, 115, 115, 117, 112, 115, 118, 114, 118, 116, 115, 117]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:17.413931438Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1613,"":"downstairs","":"downstairs","session_id":"cf2150fa-3d50-4f41-9725-73610e60f044"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:17.413942448Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1613,"":"downstairs","":"downstairs","session_id":"cf2150fa-3d50-4f41-9725-73610e60f044"} {"msg":"[1]R flush_numbers[0..12]: [196, 191, 191, 195, 187, 191, 196, 190, 196, 194, 191, 195]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:17.413952878Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1613,"":"downstairs","":"downstairs","session_id":"cf2150fa-3d50-4f41-9725-73610e60f044"} {"msg":"[1]R generation[0..12]: [118, 115, 115, 117, 112, 115, 118, 114, 118, 116, 115, 117]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:17.413963277Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1613,"":"downstairs","":"downstairs","session_id":"cf2150fa-3d50-4f41-9725-73610e60f044"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:17.413974087Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1613,"":"downstairs","":"downstairs","session_id":"cf2150fa-3d50-4f41-9725-73610e60f044"} {"msg":"[2]R flush_numbers[0..12]: [196, 191, 191, 195, 187, 191, 196, 190, 196, 194, 191, 195]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:17.413984687Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1613,"":"downstairs","":"downstairs","session_id":"cf2150fa-3d50-4f41-9725-73610e60f044"} {"msg":"[2]R generation[0..12]: [118, 115, 115, 117, 112, 115, 118, 114, 118, 116, 115, 117]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:17.414009816Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1613,"":"downstairs","":"downstairs","session_id":"cf2150fa-3d50-4f41-9725-73610e60f044"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:17.414020666Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1613,"":"downstairs","":"downstairs","session_id":"cf2150fa-3d50-4f41-9725-73610e60f044"} {"msg":"Max found gen is 119","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:17.414029975Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1613,"":"downstairs","":"downstairs","session_id":"cf2150fa-3d50-4f41-9725-73610e60f044"} {"msg":"Generation requested: 120 >= found:119","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:17.414039705Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1613,"":"downstairs","":"downstairs","session_id":"cf2150fa-3d50-4f41-9725-73610e60f044"} {"msg":"Next flush: 197","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:17.414048945Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1613,"":"downstairs","":"downstairs","session_id":"cf2150fa-3d50-4f41-9725-73610e60f044"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:17.414058175Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1613,"":"downstairs","":"downstairs","session_id":"cf2150fa-3d50-4f41-9725-73610e60f044"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:17.414067694Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1613,"session_id":"cf2150fa-3d50-4f41-9725-73610e60f044"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:17.414076154Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1613,"client":"0","":"downstairs","session_id":"cf2150fa-3d50-4f41-9725-73610e60f044"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:17.414085764Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1613,"client":"1","":"downstairs","session_id":"cf2150fa-3d50-4f41-9725-73610e60f044"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:17.414095124Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1613,"client":"2","":"downstairs","session_id":"cf2150fa-3d50-4f41-9725-73610e60f044"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:17.414104823Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1613,"session_id":"cf2150fa-3d50-4f41-9725-73610e60f044"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:17.414113773Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1613,"session_id":"cf2150fa-3d50-4f41-9725-73610e60f044"} {"msg":"b668c33e-b528-4f7b-aa35-21520cb418ca is now active with session: cf2150fa-3d50-4f41-9725-73610e60f044","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:17.414123493Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1613,"session_id":"cf2150fa-3d50-4f41-9725-73610e60f044"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:17.414132042Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1613,"session_id":"cf2150fa-3d50-4f41-9725-73610e60f044"} {"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:17.414140932Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1613} {"msg":"Activated sub_volume b668c33e-b528-4f7b-aa35-21520cb418ca","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:17.414148772Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1613} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:17.414158202Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1613,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:17.414472973Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1613,"task":"crutest"} 01/30 Write block 292 len 40960 data: 10 12 12 12 12 13 10 10 11 12 02/30 Write block 463 len 20480 data: 5 4 2 3 4 03/30 Write block 333 len 40960 data: 9 10 10 11 12 8 7 7 9 8 04/30 Read block 269 len 28672 05/30 Read block 409 len 32768 06/30 Read block 586 len 32768 07/30 Flush 08/30 Write block 139 len 40960 data: 10 10 13 12 11 11 10 11 10 9 09/30 Write block 538 len 12288 data: 17 16 15 10/30 Read block 158 len 24576 11/30 Read block 12 len 4096 12/30 Read block 149 len 8192 13/30 Read block 342 len 16384 14/30 Read block 571 len 24576 15/30 Write block 121 len 28672 data: 10 12 13 10 9 9 11 16/30 Read block 87 len 20480 17/30 Read block 550 len 40960 18/30 Read block 327 len 28672 19/30 Read block 136 len 12288 20/30 Write block 552 len 28672 data: 9 10 7 6 5 8 11 21/30 Read block 377 len 24576 22/30 Read block 94 len 12288 23/30 Read block 344 len 12288 24/30 Write block 578 len 24576 data: 11 9 9 8 11 11 25/30 Write block 311 len 4096 data: 7 26/30 Write block 92 len 28672 data: 5 7 10 10 11 11 10 27/30 Read block 234 len 40960 28/30 Write block 29 len 12288 data: 6 7 10 29/30 Read block 494 len 8192 30/30 Write block 155 len 8192 data: 6 7 ---------------------------------------------------------------- Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Write 5 Done Done Done false 1002 Acked Write 10 Done Done Done false 1003 Acked Read 7 Done Done Done false 1004 Acked Read 8 Done Done Done false 1005 Acked Read 8 Done Done Sent false 1006 Acked Flush 0 Done Done Sent false 1007 Acked Write 10 Done Done Sent false 1008 Acked Write 3 Done Done Sent false 1009 Acked Read 6 Done Done Sent false 1010 Acked Read 1 Done Done Sent false 1011 Acked Read 2 Done Done Sent false 1012 Acked Read 4 Done Done Sent false 1013 Acked Read 6 Done Done Sent false 1014 Acked Write 7 Done Done Sent false 1015 Acked Read 5 Done Done Sent false 1016 Acked Read 10 Done Done Sent false 1017 Acked Read 7 Done Done Sent false 1018 Acked Read 3 Done Done Sent false 1019 Acked Write 7 Done Done Sent false 1020 Acked Read 6 Done Done Sent false 1021 Acked Read 3 Done Done Sent false 1022 Acked Read 3 Done Done Sent false 1023 Acked Write 6 Done Done Sent false 1024 Acked Write 1 Done Done Sent false 1025 Acked Write 7 Done Done Sent false 1026 Acked Read 10 Done Done Sent false 1027 Acked Write 3 Done Done Sent false 1028 Acked Read 2 Done Done Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 25 27 Done 29 29 5 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1006)) Some(JobId(1006)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:46:17.423179242Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1613} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:17.423362146Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1613,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 196 196 196 F F F 1 020-039 115 115 115 191 191 191 T T F <--- 2 040-059 115 115 115 191 191 191 F F F 3 060-079 117 117 117 195 195 195 F F F 4 080-099 112 112 112 187 187 187 T T F <--- 5 100-119 115 115 115 191 191 191 F F F 6 120-139 118 118 118 196 196 196 T T F <--- 7 140-159 114 114 114 190 190 190 T T F <--- 8 160-179 118 118 118 196 196 196 F F F 9 180-199 116 116 116 194 194 194 F F F 10 200-219 115 115 115 191 191 191 F F F 11 220-239 117 117 117 195 195 195 F F F 12 240-259 115 115 115 191 191 191 F F F 13 260-279 118 118 118 196 196 196 F F F 14 280-299 120 120 118 197 197 196 F F T <--- 15 300-319 120 120 118 197 197 196 T T T <--- 16 320-339 120 120 118 197 197 196 F F T <--- 17 340-359 120 120 117 197 197 195 F F T <--- 18 360-379 116 116 116 192 192 192 F F F 19 380-399 112 112 112 187 187 187 F F F 20 400-419 118 118 118 196 196 196 F F F 21 420-439 118 118 118 196 196 196 F F F 22 440-459 118 118 118 196 196 196 F F F 23 460-479 120 120 112 197 197 187 F F T <--- 24 480-499 115 115 115 191 191 191 F F F 25 500-519 114 114 114 190 190 190 F F F 26 520-539 118 118 118 196 196 196 T T F <--- 27 540-559 118 118 118 196 196 196 T T F <--- 28 560-579 106 106 106 177 177 177 T T F <--- 29 580-599 118 118 118 196 196 196 T T F <--- Max gen: 120, Max flush: 197 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 60 Verifying data now No change after verify Loop: 60 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 196 196 196 F F F 1 020-039 121 121 121 198 198 198 F F F 2 040-059 115 115 115 191 191 191 F F F 3 060-079 117 117 117 195 195 195 F F F 4 080-099 121 121 121 198 198 198 F F F 5 100-119 115 115 115 191 191 191 F F F 6 120-139 121 121 121 198 198 198 F F F 7 140-159 121 121 121 198 198 198 F F F 8 160-179 118 118 118 196 196 196 F F F 9 180-199 116 116 116 194 194 194 F F F 10 200-219 115 115 115 191 191 191 F F F 11 220-239 117 117 117 195 195 195 F F F 12 240-259 115 115 115 191 191 191 F F F 13 260-279 118 118 118 196 196 196 F F F 14 280-299 120 120 120 197 197 197 F F F 15 300-319 121 121 121 198 198 198 F F F 16 320-339 120 120 120 197 197 197 F F F 17 340-359 120 120 120 197 197 197 F F F 18 360-379 116 116 116 192 192 192 F F F 19 380-399 112 112 112 187 187 187 F F F 20 400-419 118 118 118 196 196 196 F F F 21 420-439 118 118 118 196 196 196 F F F 22 440-459 118 118 118 196 196 196 F F F 23 460-479 120 120 120 197 197 197 F F F 24 480-499 115 115 115 191 191 191 F F F 25 500-519 114 114 114 190 190 190 F F F 26 520-539 121 121 121 198 198 198 F F F 27 540-559 121 121 121 198 198 198 F F F 28 560-579 121 121 121 198 198 198 F F F 29 580-599 121 121 121 198 198 198 F F F Max gen: 121, Max flush: 198 Begin loop 61 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:21.673956389Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1621,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:21.681952079Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1621,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:21.682217031Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1621} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:21.68224636Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1621} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:21.68225534Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1621} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:21.68226684Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1621} {"msg":"Crucible 7fa1282e-0b90-49af-b85c-e5d7a451a1cb has session id: 4135ba95-6d7b-4fb2-ac47-4568163e9de6","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:21.682644009Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1621,"session_id":"4135ba95-6d7b-4fb2-ac47-4568163e9de6"} {"msg":"Upstairs opts: Upstairs UUID: 7fa1282e-0b90-49af-b85c-e5d7a451a1cb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:21.682672758Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1621,"session_id":"4135ba95-6d7b-4fb2-ac47-4568163e9de6"} {"msg":"Crucible stats registered with UUID: 7fa1282e-0b90-49af-b85c-e5d7a451a1cb","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:21.682683788Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1621,"session_id":"4135ba95-6d7b-4fb2-ac47-4568163e9de6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:46:21.682692887Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1621,"session_id":"4135ba95-6d7b-4fb2-ac47-4568163e9de6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:21.682824714Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1621,"":"io task","client":"0","":"downstairs","session_id":"4135ba95-6d7b-4fb2-ac47-4568163e9de6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:21.682845223Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1621,"":"io task","client":"1","":"downstairs","session_id":"4135ba95-6d7b-4fb2-ac47-4568163e9de6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:21.682857363Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1621,"":"io task","client":"2","":"downstairs","session_id":"4135ba95-6d7b-4fb2-ac47-4568163e9de6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:21.683011208Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1621,"task":"crutest"} {"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:21.683025578Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1621} {"msg":"7fa1282e-0b90-49af-b85c-e5d7a451a1cb active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:21.683050717Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1621,"session_id":"4135ba95-6d7b-4fb2-ac47-4568163e9de6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:21.683060167Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1621,"client":"0","":"downstairs","session_id":"4135ba95-6d7b-4fb2-ac47-4568163e9de6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:21.683069727Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1621,"client":"1","":"downstairs","session_id":"4135ba95-6d7b-4fb2-ac47-4568163e9de6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:21.683079266Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1621,"client":"2","":"downstairs","session_id":"4135ba95-6d7b-4fb2-ac47-4568163e9de6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:21.683088756Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1621,"":"io task","client":"2","":"downstairs","session_id":"4135ba95-6d7b-4fb2-ac47-4568163e9de6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:21.683098626Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1621,"":"io task","client":"0","":"downstairs","session_id":"4135ba95-6d7b-4fb2-ac47-4568163e9de6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:21.683108585Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1621,"":"io task","client":"1","":"downstairs","session_id":"4135ba95-6d7b-4fb2-ac47-4568163e9de6"} {"msg":"ds_connection connected from Ok(127.0.0.1:41120)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:21.683178363Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1621,"":"io task","client":"1","":"downstairs","session_id":"4135ba95-6d7b-4fb2-ac47-4568163e9de6"} {"msg":"ds_connection connected from Ok(127.0.0.1:39873)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:21.683195333Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1621,"":"io task","client":"2","":"downstairs","session_id":"4135ba95-6d7b-4fb2-ac47-4568163e9de6"} {"msg":"ds_connection connected from Ok(127.0.0.1:64234)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:21.683205913Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1621,"":"io task","client":"0","":"downstairs","session_id":"4135ba95-6d7b-4fb2-ac47-4568163e9de6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:21.683876713Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1621,"client":"0","":"downstairs","session_id":"4135ba95-6d7b-4fb2-ac47-4568163e9de6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:21.683902533Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1621,"client":"2","":"downstairs","session_id":"4135ba95-6d7b-4fb2-ac47-4568163e9de6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:21.683913912Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1621,"client":"1","":"downstairs","session_id":"4135ba95-6d7b-4fb2-ac47-4568163e9de6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:21.684205864Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1621,"session_id":"4135ba95-6d7b-4fb2-ac47-4568163e9de6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:21.684251792Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1621,"session_id":"4135ba95-6d7b-4fb2-ac47-4568163e9de6"} {"msg":"[0]R flush_numbers[0..12]: [196, 198, 191, 195, 198, 191, 198, 198, 196, 194, 191, 195]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:21.684296481Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1621,"":"downstairs","":"downstairs","session_id":"4135ba95-6d7b-4fb2-ac47-4568163e9de6"} {"msg":"[0]R generation[0..12]: [118, 121, 115, 117, 121, 115, 121, 121, 118, 116, 115, 117]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:21.684307701Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1621,"":"downstairs","":"downstairs","session_id":"4135ba95-6d7b-4fb2-ac47-4568163e9de6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:21.684318631Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1621,"":"downstairs","":"downstairs","session_id":"4135ba95-6d7b-4fb2-ac47-4568163e9de6"} {"msg":"[1]R flush_numbers[0..12]: [196, 198, 191, 195, 198, 191, 198, 198, 196, 194, 191, 195]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:21.68432916Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1621,"":"downstairs","":"downstairs","session_id":"4135ba95-6d7b-4fb2-ac47-4568163e9de6"} {"msg":"[1]R generation[0..12]: [118, 121, 115, 117, 121, 115, 121, 121, 118, 116, 115, 117]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:21.68433995Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1621,"":"downstairs","":"downstairs","session_id":"4135ba95-6d7b-4fb2-ac47-4568163e9de6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:21.68435089Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1621,"":"downstairs","":"downstairs","session_id":"4135ba95-6d7b-4fb2-ac47-4568163e9de6"} {"msg":"[2]R flush_numbers[0..12]: [196, 198, 191, 195, 198, 191, 198, 198, 196, 194, 191, 195]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:21.684361319Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1621,"":"downstairs","":"downstairs","session_id":"4135ba95-6d7b-4fb2-ac47-4568163e9de6"} {"msg":"[2]R generation[0..12]: [118, 121, 115, 117, 121, 115, 121, 121, 118, 116, 115, 117]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:21.684371799Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1621,"":"downstairs","":"downstairs","session_id":"4135ba95-6d7b-4fb2-ac47-4568163e9de6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:21.684382539Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1621,"":"downstairs","":"downstairs","session_id":"4135ba95-6d7b-4fb2-ac47-4568163e9de6"} {"msg":"Max found gen is 122","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:21.684391798Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1621,"":"downstairs","":"downstairs","session_id":"4135ba95-6d7b-4fb2-ac47-4568163e9de6"} {"msg":"Generation requested: 122 >= found:122","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:21.684401528Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1621,"":"downstairs","":"downstairs","session_id":"4135ba95-6d7b-4fb2-ac47-4568163e9de6"} {"msg":"Next flush: 199","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:21.684410868Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1621,"":"downstairs","":"downstairs","session_id":"4135ba95-6d7b-4fb2-ac47-4568163e9de6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:21.684421688Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1621,"":"downstairs","":"downstairs","session_id":"4135ba95-6d7b-4fb2-ac47-4568163e9de6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:21.684431657Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1621,"session_id":"4135ba95-6d7b-4fb2-ac47-4568163e9de6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:21.684440517Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1621,"client":"0","":"downstairs","session_id":"4135ba95-6d7b-4fb2-ac47-4568163e9de6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:21.684450267Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1621,"client":"1","":"downstairs","session_id":"4135ba95-6d7b-4fb2-ac47-4568163e9de6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:21.684459956Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1621,"client":"2","":"downstairs","session_id":"4135ba95-6d7b-4fb2-ac47-4568163e9de6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:21.684469946Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1621,"session_id":"4135ba95-6d7b-4fb2-ac47-4568163e9de6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:21.684479086Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1621,"session_id":"4135ba95-6d7b-4fb2-ac47-4568163e9de6"} {"msg":"7fa1282e-0b90-49af-b85c-e5d7a451a1cb is now active with session: 4135ba95-6d7b-4fb2-ac47-4568163e9de6","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:21.684488996Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1621,"session_id":"4135ba95-6d7b-4fb2-ac47-4568163e9de6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:21.684497655Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1621,"session_id":"4135ba95-6d7b-4fb2-ac47-4568163e9de6"} {"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:21.684506565Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1621} {"msg":"Activated sub_volume 7fa1282e-0b90-49af-b85c-e5d7a451a1cb","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:21.684514385Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1621} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:21.684522475Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1621,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:21.684834256Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1621,"task":"crutest"} 01/30 Write block 45 len 36864 data: 6 7 7 9 8 11 9 8 7 02/30 Flush 03/30 Write block 375 len 12288 data: 8 13 13 04/30 Read block 61 len 16384 05/30 Write block 276 len 24576 data: 9 9 9 10 10 10 06/30 Flush 07/30 Write block 361 len 40960 data: 9 12 13 14 12 10 12 9 9 7 08/30 Read block 557 len 40960 09/30 Write block 485 len 36864 data: 9 10 11 12 13 14 14 12 10 10/30 Write block 90 len 4096 data: 6 11/30 Write block 485 len 28672 data: 10 11 12 13 14 15 15 12/30 Read block 199 len 40960 13/30 Read block 276 len 40960 14/30 Read block 566 len 24576 15/30 Read block 331 len 4096 16/30 Read block 489 len 28672 17/30 Write block 416 len 8192 data: 9 9 18/30 Write block 423 len 40960 data: 10 11 10 10 10 9 11 9 8 7 19/30 Flush 20/30 Write block 124 len 12288 data: 11 10 10 21/30 Read block 354 len 24576 22/30 Write block 114 len 12288 data: 9 10 11 23/30 Read block 295 len 8192 24/30 Read block 538 len 12288 25/30 Read block 578 len 12288 26/30 Read block 358 len 40960 27/30 Read block 551 len 28672 28/30 Read block 180 len 12288 29/30 Write block 298 len 4096 data: 11 30/30 Read block 173 len 16384 ---------------------------------------------------------------- Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 3 Done Done Done false 1003 Acked Read 4 Done Done Done false 1004 Acked Write 6 Done Done Done false 1005 Acked Flush 0 Done Sent Done false 1006 Acked Write 10 Done Sent Done false 1007 Acked Read 10 Done Sent Done false 1008 Acked Write 9 Done Sent Done false 1009 Acked Write 1 Done Sent Done false 1010 Acked Write 7 Done Sent Done false 1011 Acked Read 10 Done Sent Done false 1012 Acked Read 10 Done Sent Done false 1013 Acked Read 6 Done Sent Done false 1014 Acked Read 1 Done Sent Done false 1015 Acked Read 7 Done Sent Done false 1016 Acked Write 2 Done Sent Done false 1017 Acked Write 10 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Write 3 Done Sent Done false 1020 Acked Read 6 Done Sent Done false 1021 Acked Write 3 Done Sent Done false 1022 Acked Read 2 Done Sent Done false 1023 Acked Read 3 Done Sent Done false 1024 Acked Read 3 Done Sent Done false 1025 Acked Read 10 Done Sent Done false 1026 Acked Read 7 Done Sent Done false 1027 Acked Read 3 Done Sent Done false 1028 Acked Write 1 Done Sent Done false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 25 1 26 Done 28 3 27 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1001)) Some(JobId(1018)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:46:21.693736759Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1621} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:21.693916364Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1621,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 196 196 196 F F F 1 020-039 121 121 121 198 198 198 F F F 2 040-059 122 122 122 199 199 199 F F F 3 060-079 117 117 117 195 195 195 F F F 4 080-099 122 121 122 201 198 201 F F F <--- 5 100-119 115 115 115 191 191 191 T F T <--- 6 120-139 121 121 121 198 198 198 T F T <--- 7 140-159 121 121 121 198 198 198 F F F 8 160-179 118 118 118 196 196 196 F F F 9 180-199 116 116 116 194 194 194 F F F 10 200-219 115 115 115 191 191 191 F F F 11 220-239 117 117 117 195 195 195 F F F 12 240-259 115 115 115 191 191 191 F F F 13 260-279 122 118 122 200 196 200 F T F <--- 14 280-299 122 120 122 200 197 200 T T T <--- 15 300-319 121 121 121 198 198 198 F F F 16 320-339 120 120 120 197 197 197 F F F 17 340-359 120 120 120 197 197 197 F F F 18 360-379 122 116 122 201 192 201 F T F <--- 19 380-399 112 112 112 187 187 187 F F F 20 400-419 122 118 122 201 196 201 F F F <--- 21 420-439 122 118 122 201 196 201 F F F <--- 22 440-459 118 118 118 196 196 196 F F F 23 460-479 120 120 120 197 197 197 F F F 24 480-499 122 115 122 201 191 201 F F F <--- 25 500-519 114 114 114 190 190 190 F F F 26 520-539 121 121 121 198 198 198 F F F 27 540-559 121 121 121 198 198 198 F F F 28 560-579 121 121 121 198 198 198 F F F 29 580-599 121 121 121 198 198 198 F F F Max gen: 122, Max flush: 201 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 61 Verifying data now No change after verify Loop: 61 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 196 196 196 F F F 1 020-039 121 121 121 198 198 198 F F F 2 040-059 122 122 122 199 199 199 F F F 3 060-079 117 117 117 195 195 195 F F F 4 080-099 122 122 122 201 201 201 F F F 5 100-119 123 123 123 202 202 202 F F F 6 120-139 123 123 123 202 202 202 F F F 7 140-159 121 121 121 198 198 198 F F F 8 160-179 118 118 118 196 196 196 F F F 9 180-199 116 116 116 194 194 194 F F F 10 200-219 115 115 115 191 191 191 F F F 11 220-239 117 117 117 195 195 195 F F F 12 240-259 115 115 115 191 191 191 F F F 13 260-279 122 122 122 200 200 200 F F F 14 280-299 123 123 123 202 202 202 F F F 15 300-319 121 121 121 198 198 198 F F F 16 320-339 120 120 120 197 197 197 F F F 17 340-359 120 120 120 197 197 197 F F F 18 360-379 122 122 122 201 201 201 F F F 19 380-399 112 112 112 187 187 187 F F F 20 400-419 122 122 122 201 201 201 F F F 21 420-439 122 122 122 201 201 201 F F F 22 440-459 118 118 118 196 196 196 F F F 23 460-479 120 120 120 197 197 197 F F F 24 480-499 122 122 122 201 201 201 F F F 25 500-519 114 114 114 190 190 190 F F F 26 520-539 121 121 121 198 198 198 F F F 27 540-559 121 121 121 198 198 198 F F F 28 560-579 121 121 121 198 198 198 F F F 29 580-599 121 121 121 198 198 198 F F F Max gen: 123, Max flush: 202 Begin loop 62 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:25.935493517Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1629,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:25.943467147Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1629,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:25.943757668Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1629} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:25.943782677Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1629} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:25.943794077Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1629} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:25.943805487Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1629} {"msg":"Crucible b80221e4-00ef-49f9-af2a-3209e83cafc5 has session id: 47c64952-485c-409d-bd2d-ce7b6f28e2c5","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:25.944163526Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1629,"session_id":"47c64952-485c-409d-bd2d-ce7b6f28e2c5"} {"msg":"Upstairs opts: Upstairs UUID: b80221e4-00ef-49f9-af2a-3209e83cafc5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:25.944182736Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1629,"session_id":"47c64952-485c-409d-bd2d-ce7b6f28e2c5"} {"msg":"Crucible stats registered with UUID: b80221e4-00ef-49f9-af2a-3209e83cafc5","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:25.944192326Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1629,"session_id":"47c64952-485c-409d-bd2d-ce7b6f28e2c5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:46:25.944201215Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1629,"session_id":"47c64952-485c-409d-bd2d-ce7b6f28e2c5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:25.944344991Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1629,"":"io task","client":"0","":"downstairs","session_id":"47c64952-485c-409d-bd2d-ce7b6f28e2c5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:25.94437866Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1629,"":"io task","client":"2","":"downstairs","session_id":"47c64952-485c-409d-bd2d-ce7b6f28e2c5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:25.94438974Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1629,"":"io task","client":"1","":"downstairs","session_id":"47c64952-485c-409d-bd2d-ce7b6f28e2c5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:25.944515566Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1629,"task":"crutest"} {"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:25.944533066Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1629} {"msg":"b80221e4-00ef-49f9-af2a-3209e83cafc5 active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:25.944557805Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1629,"session_id":"47c64952-485c-409d-bd2d-ce7b6f28e2c5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:25.944567145Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1629,"client":"0","":"downstairs","session_id":"47c64952-485c-409d-bd2d-ce7b6f28e2c5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:25.944576845Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1629,"client":"1","":"downstairs","session_id":"47c64952-485c-409d-bd2d-ce7b6f28e2c5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:25.944586484Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1629,"client":"2","":"downstairs","session_id":"47c64952-485c-409d-bd2d-ce7b6f28e2c5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:25.944598244Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1629,"":"io task","client":"2","":"downstairs","session_id":"47c64952-485c-409d-bd2d-ce7b6f28e2c5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:25.944608404Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1629,"":"io task","client":"0","":"downstairs","session_id":"47c64952-485c-409d-bd2d-ce7b6f28e2c5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:25.944618433Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1629,"":"io task","client":"1","":"downstairs","session_id":"47c64952-485c-409d-bd2d-ce7b6f28e2c5"} {"msg":"ds_connection connected from Ok(127.0.0.1:53034)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:25.944680162Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1629,"":"io task","client":"2","":"downstairs","session_id":"47c64952-485c-409d-bd2d-ce7b6f28e2c5"} {"msg":"ds_connection connected from Ok(127.0.0.1:54355)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:25.944704841Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1629,"":"io task","client":"1","":"downstairs","session_id":"47c64952-485c-409d-bd2d-ce7b6f28e2c5"} {"msg":"ds_connection connected from Ok(127.0.0.1:34019)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:25.944715761Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1629,"":"io task","client":"0","":"downstairs","session_id":"47c64952-485c-409d-bd2d-ce7b6f28e2c5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:25.945457249Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1629,"client":"0","":"downstairs","session_id":"47c64952-485c-409d-bd2d-ce7b6f28e2c5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:25.945502418Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1629,"client":"2","":"downstairs","session_id":"47c64952-485c-409d-bd2d-ce7b6f28e2c5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:25.945516247Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1629,"client":"1","":"downstairs","session_id":"47c64952-485c-409d-bd2d-ce7b6f28e2c5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:25.94577667Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1629,"session_id":"47c64952-485c-409d-bd2d-ce7b6f28e2c5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:25.945818449Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1629,"session_id":"47c64952-485c-409d-bd2d-ce7b6f28e2c5"} {"msg":"[0]R flush_numbers[0..12]: [196, 198, 199, 195, 201, 202, 202, 198, 196, 194, 191, 195]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:25.945857698Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1629,"":"downstairs","":"downstairs","session_id":"47c64952-485c-409d-bd2d-ce7b6f28e2c5"} {"msg":"[0]R generation[0..12]: [118, 121, 122, 117, 122, 123, 123, 121, 118, 116, 115, 117]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:25.945869067Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1629,"":"downstairs","":"downstairs","session_id":"47c64952-485c-409d-bd2d-ce7b6f28e2c5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:25.945880107Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1629,"":"downstairs","":"downstairs","session_id":"47c64952-485c-409d-bd2d-ce7b6f28e2c5"} {"msg":"[1]R flush_numbers[0..12]: [196, 198, 199, 195, 201, 202, 202, 198, 196, 194, 191, 195]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:25.945892777Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1629,"":"downstairs","":"downstairs","session_id":"47c64952-485c-409d-bd2d-ce7b6f28e2c5"} {"msg":"[1]R generation[0..12]: [118, 121, 122, 117, 122, 123, 123, 121, 118, 116, 115, 117]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:25.945903426Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1629,"":"downstairs","":"downstairs","session_id":"47c64952-485c-409d-bd2d-ce7b6f28e2c5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:25.945914236Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1629,"":"downstairs","":"downstairs","session_id":"47c64952-485c-409d-bd2d-ce7b6f28e2c5"} {"msg":"[2]R flush_numbers[0..12]: [196, 198, 199, 195, 201, 202, 202, 198, 196, 194, 191, 195]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:25.945924806Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1629,"":"downstairs","":"downstairs","session_id":"47c64952-485c-409d-bd2d-ce7b6f28e2c5"} {"msg":"[2]R generation[0..12]: [118, 121, 122, 117, 122, 123, 123, 121, 118, 116, 115, 117]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:25.945935335Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1629,"":"downstairs","":"downstairs","session_id":"47c64952-485c-409d-bd2d-ce7b6f28e2c5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:25.945946185Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1629,"":"downstairs","":"downstairs","session_id":"47c64952-485c-409d-bd2d-ce7b6f28e2c5"} {"msg":"Max found gen is 124","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:25.945955705Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1629,"":"downstairs","":"downstairs","session_id":"47c64952-485c-409d-bd2d-ce7b6f28e2c5"} {"msg":"Generation requested: 124 >= found:124","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:25.945965344Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1629,"":"downstairs","":"downstairs","session_id":"47c64952-485c-409d-bd2d-ce7b6f28e2c5"} {"msg":"Next flush: 203","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:25.945974634Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1629,"":"downstairs","":"downstairs","session_id":"47c64952-485c-409d-bd2d-ce7b6f28e2c5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:25.945984104Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1629,"":"downstairs","":"downstairs","session_id":"47c64952-485c-409d-bd2d-ce7b6f28e2c5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:25.945993764Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1629,"session_id":"47c64952-485c-409d-bd2d-ce7b6f28e2c5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:25.946002413Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1629,"client":"0","":"downstairs","session_id":"47c64952-485c-409d-bd2d-ce7b6f28e2c5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:25.946011933Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1629,"client":"1","":"downstairs","session_id":"47c64952-485c-409d-bd2d-ce7b6f28e2c5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:25.946021763Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1629,"client":"2","":"downstairs","session_id":"47c64952-485c-409d-bd2d-ce7b6f28e2c5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:25.946031483Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1629,"session_id":"47c64952-485c-409d-bd2d-ce7b6f28e2c5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:25.946041852Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1629,"session_id":"47c64952-485c-409d-bd2d-ce7b6f28e2c5"} {"msg":"b80221e4-00ef-49f9-af2a-3209e83cafc5 is now active with session: 47c64952-485c-409d-bd2d-ce7b6f28e2c5","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:25.946051762Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1629,"session_id":"47c64952-485c-409d-bd2d-ce7b6f28e2c5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:25.946060372Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1629,"session_id":"47c64952-485c-409d-bd2d-ce7b6f28e2c5"} {"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:25.946069461Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1629} {"msg":"Activated sub_volume b80221e4-00ef-49f9-af2a-3209e83cafc5","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:25.946077331Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1629} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:25.946085361Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1629,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:25.946436051Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1629,"task":"crutest"} 01/30 Flush 02/30 Write block 411 len 28672 data: 12 12 12 9 6 10 10 03/30 Read block 260 len 40960 04/30 Write block 449 len 24576 data: 4 6 6 5 5 6 05/30 Read block 242 len 4096 06/30 Write block 400 len 24576 data: 11 11 13 12 13 13 07/30 Write block 355 len 40960 data: 9 9 10 9 8 9 10 13 14 15 08/30 Read block 265 len 40960 09/30 Write block 82 len 4096 data: 7 10/30 Write block 573 len 16384 data: 10 10 9 11 11/30 Read block 55 len 32768 12/30 Write block 535 len 40960 data: 15 13 16 18 17 16 12 11 12 11 13/30 Write block 158 len 4096 data: 10 14/30 Read block 261 len 12288 15/30 Read block 64 len 40960 16/30 Write block 42 len 8192 data: 5 4 17/30 Read block 158 len 32768 18/30 Write block 90 len 8192 data: 7 4 19/30 Write block 581 len 24576 data: 9 12 12 11 11 13 20/30 Read block 429 len 28672 21/30 Read block 239 len 36864 22/30 Write block 193 len 8192 data: 9 11 23/30 Write block 15 len 32768 data: 8 9 9 9 6 6 5 10 24/30 Flush 25/30 Write block 575 len 12288 data: 10 12 10 26/30 Read block 332 len 32768 27/30 Write block 442 len 36864 data: 7 6 7 6 6 5 4 5 7 28/30 Write block 130 len 36864 data: 10 10 9 10 10 10 12 9 9 29/30 Read block 316 len 40960 30/30 Read block 282 len 4096 ---------------------------------------------------------------- Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 7 Done Sent Done false 1002 Acked Read 10 Done Sent Done false 1003 Acked Write 6 Done Sent Done false 1004 Acked Read 1 Done Sent Done false 1005 Acked Write 6 Done Sent Done false 1006 Acked Write 10 Done Sent Done false 1007 Acked Read 10 Done Sent Done false 1008 Acked Write 1 Done Sent Done false 1009 Acked Write 4 Done Sent Done false 1010 Acked Read 8 Done Sent Done false 1011 Acked Write 10 Done Sent Done false 1012 Acked Write 1 Done Sent Done false 1013 Acked Read 3 Done Sent Done false 1014 Acked Read 10 Done Sent Done false 1015 Acked Write 2 Done Sent Done false 1016 Acked Read 8 Done Sent Done false 1017 Acked Write 2 Done Sent Done false 1018 Acked Write 6 Done Sent Done false 1019 Acked Read 7 Done Sent Done false 1020 Acked Read 9 Done Sent Done false 1021 Acked Write 2 Done Sent Done false 1022 Acked Write 8 Done Sent Done false 1023 Acked Flush 0 Done Sent Done false 1024 Acked Write 3 Done Sent Done false 1025 Acked Read 8 Done Sent Done false 1026 Acked Write 9 Done Sent Done false 1027 Acked Write 9 Done Sent Done false 1028 Acked Read 10 Done Sent Done false 1029 Acked Read 1 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 29 0 30 Done 28 0 29 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1000)) Some(JobId(1023)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:46:25.955789241Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1629} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:25.955971756Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1629,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 118 124 204 196 204 F F F <--- 1 020-039 124 121 124 204 198 204 F F F <--- 2 040-059 124 122 124 204 199 204 F F F <--- 3 060-079 117 117 117 195 195 195 F F F 4 080-099 124 122 124 204 201 204 F F F <--- 5 100-119 123 123 123 202 202 202 F F F 6 120-139 123 123 123 202 202 202 T F T <--- 7 140-159 124 121 124 204 198 204 F F F <--- 8 160-179 118 118 118 196 196 196 F F F 9 180-199 124 116 124 204 194 204 F F F <--- 10 200-219 115 115 115 191 191 191 F F F 11 220-239 117 117 117 195 195 195 F F F 12 240-259 115 115 115 191 191 191 F F F 13 260-279 122 122 122 200 200 200 F F F 14 280-299 123 123 123 202 202 202 F F F 15 300-319 121 121 121 198 198 198 F F F 16 320-339 120 120 120 197 197 197 F F F 17 340-359 124 120 124 204 197 204 F F F <--- 18 360-379 124 122 124 204 201 204 F F F <--- 19 380-399 112 112 112 187 187 187 F F F 20 400-419 124 122 124 204 201 204 F F F <--- 21 420-439 122 122 122 201 201 201 F F F 22 440-459 124 118 124 204 196 204 T F T <--- 23 460-479 120 120 120 197 197 197 F F F 24 480-499 122 122 122 201 201 201 F F F 25 500-519 114 114 114 190 190 190 F F F 26 520-539 124 121 124 204 198 204 F F F <--- 27 540-559 124 121 124 204 198 204 F F F <--- 28 560-579 124 121 124 204 198 204 T F T <--- 29 580-599 124 121 124 204 198 204 F F F <--- Max gen: 124, Max flush: 204 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 62 Verifying data now No change after verify Loop: 62 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 124 124 204 204 204 F F F 1 020-039 124 124 124 204 204 204 F F F 2 040-059 124 124 124 204 204 204 F F F 3 060-079 117 117 117 195 195 195 F F F 4 080-099 124 124 124 204 204 204 F F F 5 100-119 123 123 123 202 202 202 F F F 6 120-139 125 125 125 205 205 205 F F F 7 140-159 124 124 124 204 204 204 F F F 8 160-179 118 118 118 196 196 196 F F F 9 180-199 124 124 124 204 204 204 F F F 10 200-219 115 115 115 191 191 191 F F F 11 220-239 117 117 117 195 195 195 F F F 12 240-259 115 115 115 191 191 191 F F F 13 260-279 122 122 122 200 200 200 F F F 14 280-299 123 123 123 202 202 202 F F F 15 300-319 121 121 121 198 198 198 F F F 16 320-339 120 120 120 197 197 197 F F F 17 340-359 124 124 124 204 204 204 F F F 18 360-379 124 124 124 204 204 204 F F F 19 380-399 112 112 112 187 187 187 F F F 20 400-419 124 124 124 204 204 204 F F F 21 420-439 122 122 122 201 201 201 F F F 22 440-459 125 125 125 205 205 205 F F F 23 460-479 120 120 120 197 197 197 F F F 24 480-499 122 122 122 201 201 201 F F F 25 500-519 114 114 114 190 190 190 F F F 26 520-539 124 124 124 204 204 204 F F F 27 540-559 124 124 124 204 204 204 F F F 28 560-579 125 125 125 205 205 205 F F F 29 580-599 124 124 124 204 204 204 F F F Max gen: 125, Max flush: 205 Begin loop 63 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:30.207452893Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1637,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:30.215430163Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1637,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:30.215716615Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1637} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:30.215740194Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1637} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:30.215748634Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1637} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:30.215760144Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1637} {"msg":"Crucible af0242fc-5176-41ab-8e2e-f33533d23bcb has session id: 92341f45-e6dc-4abf-b1a9-750587247634","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:30.216102254Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1637,"session_id":"92341f45-e6dc-4abf-b1a9-750587247634"} {"msg":"Upstairs opts: Upstairs UUID: af0242fc-5176-41ab-8e2e-f33533d23bcb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:30.216131593Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1637,"session_id":"92341f45-e6dc-4abf-b1a9-750587247634"} {"msg":"Crucible stats registered with UUID: af0242fc-5176-41ab-8e2e-f33533d23bcb","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:30.216144693Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1637,"session_id":"92341f45-e6dc-4abf-b1a9-750587247634"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:46:30.216153832Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1637,"session_id":"92341f45-e6dc-4abf-b1a9-750587247634"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:30.216269689Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1637,"":"io task","client":"0","":"downstairs","session_id":"92341f45-e6dc-4abf-b1a9-750587247634"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:30.216287429Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1637,"":"io task","client":"2","":"downstairs","session_id":"92341f45-e6dc-4abf-b1a9-750587247634"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:30.216297438Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1637,"":"io task","client":"1","":"downstairs","session_id":"92341f45-e6dc-4abf-b1a9-750587247634"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:30.216447884Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1637,"task":"crutest"} {"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:30.216464644Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1637} {"msg":"af0242fc-5176-41ab-8e2e-f33533d23bcb active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:30.216500042Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1637,"session_id":"92341f45-e6dc-4abf-b1a9-750587247634"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:30.216509512Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1637,"client":"0","":"downstairs","session_id":"92341f45-e6dc-4abf-b1a9-750587247634"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:30.216519232Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1637,"client":"1","":"downstairs","session_id":"92341f45-e6dc-4abf-b1a9-750587247634"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:30.216528972Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1637,"client":"2","":"downstairs","session_id":"92341f45-e6dc-4abf-b1a9-750587247634"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:30.216538581Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1637,"":"io task","client":"2","":"downstairs","session_id":"92341f45-e6dc-4abf-b1a9-750587247634"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:30.216548601Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1637,"":"io task","client":"0","":"downstairs","session_id":"92341f45-e6dc-4abf-b1a9-750587247634"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:30.216558511Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1637,"":"io task","client":"1","":"downstairs","session_id":"92341f45-e6dc-4abf-b1a9-750587247634"} {"msg":"ds_connection connected from Ok(127.0.0.1:51230)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:30.216655048Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1637,"":"io task","client":"1","":"downstairs","session_id":"92341f45-e6dc-4abf-b1a9-750587247634"} {"msg":"ds_connection connected from Ok(127.0.0.1:65433)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:30.216680547Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1637,"":"io task","client":"0","":"downstairs","session_id":"92341f45-e6dc-4abf-b1a9-750587247634"} {"msg":"ds_connection connected from Ok(127.0.0.1:59571)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:30.216694497Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1637,"":"io task","client":"2","":"downstairs","session_id":"92341f45-e6dc-4abf-b1a9-750587247634"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:30.217324299Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1637,"client":"0","":"downstairs","session_id":"92341f45-e6dc-4abf-b1a9-750587247634"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:30.217367907Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1637,"client":"1","":"downstairs","session_id":"92341f45-e6dc-4abf-b1a9-750587247634"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:30.217415996Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1637,"client":"2","":"downstairs","session_id":"92341f45-e6dc-4abf-b1a9-750587247634"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:30.21764231Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1637,"session_id":"92341f45-e6dc-4abf-b1a9-750587247634"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:30.217652199Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1637,"session_id":"92341f45-e6dc-4abf-b1a9-750587247634"} {"msg":"[0]R flush_numbers[0..12]: [204, 204, 204, 195, 204, 202, 205, 204, 196, 204, 191, 195]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:30.217731307Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1637,"":"downstairs","":"downstairs","session_id":"92341f45-e6dc-4abf-b1a9-750587247634"} {"msg":"[0]R generation[0..12]: [124, 124, 124, 117, 124, 123, 125, 124, 118, 124, 115, 117]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:30.217742517Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1637,"":"downstairs","":"downstairs","session_id":"92341f45-e6dc-4abf-b1a9-750587247634"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:30.217753456Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1637,"":"downstairs","":"downstairs","session_id":"92341f45-e6dc-4abf-b1a9-750587247634"} {"msg":"[1]R flush_numbers[0..12]: [204, 204, 204, 195, 204, 202, 205, 204, 196, 204, 191, 195]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:30.217764106Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1637,"":"downstairs","":"downstairs","session_id":"92341f45-e6dc-4abf-b1a9-750587247634"} {"msg":"[1]R generation[0..12]: [124, 124, 124, 117, 124, 123, 125, 124, 118, 124, 115, 117]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:30.217774616Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1637,"":"downstairs","":"downstairs","session_id":"92341f45-e6dc-4abf-b1a9-750587247634"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:30.217785465Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1637,"":"downstairs","":"downstairs","session_id":"92341f45-e6dc-4abf-b1a9-750587247634"} {"msg":"[2]R flush_numbers[0..12]: [204, 204, 204, 195, 204, 202, 205, 204, 196, 204, 191, 195]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:30.217796135Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1637,"":"downstairs","":"downstairs","session_id":"92341f45-e6dc-4abf-b1a9-750587247634"} {"msg":"[2]R generation[0..12]: [124, 124, 124, 117, 124, 123, 125, 124, 118, 124, 115, 117]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:30.217809025Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1637,"":"downstairs","":"downstairs","session_id":"92341f45-e6dc-4abf-b1a9-750587247634"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:30.217819834Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1637,"":"downstairs","":"downstairs","session_id":"92341f45-e6dc-4abf-b1a9-750587247634"} {"msg":"Max found gen is 126","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:30.217829274Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1637,"":"downstairs","":"downstairs","session_id":"92341f45-e6dc-4abf-b1a9-750587247634"} {"msg":"Generation requested: 126 >= found:126","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:30.217839004Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1637,"":"downstairs","":"downstairs","session_id":"92341f45-e6dc-4abf-b1a9-750587247634"} {"msg":"Next flush: 206","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:30.217848334Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1637,"":"downstairs","":"downstairs","session_id":"92341f45-e6dc-4abf-b1a9-750587247634"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:30.217857723Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1637,"":"downstairs","":"downstairs","session_id":"92341f45-e6dc-4abf-b1a9-750587247634"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:30.217867253Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1637,"session_id":"92341f45-e6dc-4abf-b1a9-750587247634"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:30.217876023Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1637,"client":"0","":"downstairs","session_id":"92341f45-e6dc-4abf-b1a9-750587247634"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:30.217885553Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1637,"client":"1","":"downstairs","session_id":"92341f45-e6dc-4abf-b1a9-750587247634"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:30.217895272Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1637,"client":"2","":"downstairs","session_id":"92341f45-e6dc-4abf-b1a9-750587247634"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:30.217905202Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1637,"session_id":"92341f45-e6dc-4abf-b1a9-750587247634"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:30.217914332Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1637,"session_id":"92341f45-e6dc-4abf-b1a9-750587247634"} {"msg":"af0242fc-5176-41ab-8e2e-f33533d23bcb is now active with session: 92341f45-e6dc-4abf-b1a9-750587247634","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:30.217924231Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1637,"session_id":"92341f45-e6dc-4abf-b1a9-750587247634"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:30.217932971Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1637,"session_id":"92341f45-e6dc-4abf-b1a9-750587247634"} {"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:30.217942061Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1637} {"msg":"Activated sub_volume af0242fc-5176-41ab-8e2e-f33533d23bcb","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:30.217950051Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1637} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:30.21795816Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1637,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:30.218296481Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1637,"task":"crutest"} 01/30 Write block 82 len 32768 data: 8 9 9 9 8 7 4 5 02/30 Read block 492 len 28672 03/30 Read block 402 len 28672 04/30 Read block 34 len 36864 05/30 Write block 71 len 8192 data: 10 10 06/30 Read block 333 len 4096 07/30 Read block 179 len 24576 08/30 Write block 228 len 24576 data: 14 14 14 15 10 11 09/30 Write block 198 len 4096 data: 13 10/30 Read block 305 len 16384 11/30 Write block 211 len 12288 data: 13 14 15 12/30 Read block 215 len 16384 13/30 Read block 549 len 40960 14/30 Read block 504 len 16384 15/30 Write block 533 len 40960 data: 13 13 16 14 17 19 18 17 13 12 16/30 Read block 558 len 16384 17/30 Flush 18/30 Write block 223 len 8192 data: 14 15 19/30 Read block 42 len 20480 20/30 Write block 348 len 20480 data: 9 8 6 4 5 21/30 Read block 534 len 36864 22/30 Read block 530 len 36864 23/30 Flush 24/30 Write block 436 len 24576 data: 8 5 6 7 9 8 25/30 Write block 157 len 12288 data: 7 11 13 26/30 Read block 127 len 4096 27/30 Read block 557 len 20480 28/30 Write block 448 len 12288 data: 5 6 8 29/30 Write block 313 len 40960 data: 8 9 8 7 5 4 6 5 8 8 30/30 Read block 581 len 20480 ---------------------------------------------------------------- Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Read 7 Done Done Done false 1002 Acked Read 7 Done Done Done false 1003 Acked Read 9 Done Done Sent false 1004 Acked Write 2 Done Done Sent false 1005 Acked Read 1 Done Done Sent false 1006 Acked Read 6 Done Done Sent false 1007 Acked Write 6 Done Done Sent false 1008 Acked Write 1 Done Done Sent false 1009 Acked Read 4 Done Done Sent false 1010 Acked Write 3 Done Done Sent false 1011 Acked Read 4 Done Done Sent false 1012 Acked Read 10 Done Done Sent false 1013 Acked Read 4 Done Done Sent false 1014 Acked Write 10 Done Done Sent false 1015 Acked Read 4 Done Done Sent false 1016 Acked Flush 0 Done Done Sent false 1017 Acked Write 2 Done Done Sent false 1018 Acked Read 5 Done Done Sent false 1019 Acked Write 5 Done Done Sent false 1020 Acked Read 9 Done Done Sent false 1021 Acked Read 9 Done Done Sent false 1022 Acked Flush 0 Done Done Sent false 1023 Acked Write 6 Done Done Sent false 1024 Acked Write 3 Done Done Sent false 1025 Acked Read 1 Done Done Sent false 1026 Acked Read 5 Done Done Sent false 1027 Acked Write 3 Done Done Sent false 1028 Acked Write 10 Done Done Sent false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 27 28 Done 30 29 3 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:46:30.227595932Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1637} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:30.227780547Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1637,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 124 124 204 204 204 F F F 1 020-039 124 124 124 204 204 204 F F F 2 040-059 124 124 124 204 204 204 F F F 3 060-079 126 126 117 206 206 195 F F F <--- 4 080-099 126 126 124 206 206 204 F F T <--- 5 100-119 123 123 123 202 202 202 F F F 6 120-139 125 125 125 205 205 205 F F F 7 140-159 124 124 124 204 204 204 T T F <--- 8 160-179 118 118 118 196 196 196 F F F 9 180-199 126 126 124 206 206 204 F F F <--- 10 200-219 126 126 115 206 206 191 F F F <--- 11 220-239 126 126 117 207 207 195 F F F <--- 12 240-259 115 115 115 191 191 191 F F F 13 260-279 122 122 122 200 200 200 F F F 14 280-299 123 123 123 202 202 202 F F F 15 300-319 121 121 121 198 198 198 T T F <--- 16 320-339 120 120 120 197 197 197 T T F <--- 17 340-359 126 126 124 207 207 204 F F F <--- 18 360-379 124 124 124 204 204 204 F F F 19 380-399 112 112 112 187 187 187 F F F 20 400-419 124 124 124 204 204 204 F F F 21 420-439 122 122 122 201 201 201 T T F <--- 22 440-459 125 125 125 205 205 205 T T F <--- 23 460-479 120 120 120 197 197 197 F F F 24 480-499 122 122 122 201 201 201 F F F 25 500-519 114 114 114 190 190 190 F F F 26 520-539 126 126 124 206 206 204 F F F <--- 27 540-559 126 126 124 206 206 204 F F F <--- 28 560-579 125 125 125 205 205 205 F F F 29 580-599 124 124 124 204 204 204 F F F Max gen: 126, Max flush: 207 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 63 Verifying data now No change after verify Loop: 63 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 124 124 204 204 204 F F F 1 020-039 124 124 124 204 204 204 F F F 2 040-059 124 124 124 204 204 204 F F F 3 060-079 126 126 126 206 206 206 F F F 4 080-099 126 126 126 206 206 206 F F F 5 100-119 123 123 123 202 202 202 F F F 6 120-139 125 125 125 205 205 205 F F F 7 140-159 127 127 127 208 208 208 F F F 8 160-179 118 118 118 196 196 196 F F F 9 180-199 126 126 126 206 206 206 F F F 10 200-219 126 126 126 206 206 206 F F F 11 220-239 126 126 126 207 207 207 F F F 12 240-259 115 115 115 191 191 191 F F F 13 260-279 122 122 122 200 200 200 F F F 14 280-299 123 123 123 202 202 202 F F F 15 300-319 127 127 127 208 208 208 F F F 16 320-339 127 127 127 208 208 208 F F F 17 340-359 126 126 126 207 207 207 F F F 18 360-379 124 124 124 204 204 204 F F F 19 380-399 112 112 112 187 187 187 F F F 20 400-419 124 124 124 204 204 204 F F F 21 420-439 127 127 127 208 208 208 F F F 22 440-459 127 127 127 208 208 208 F F F 23 460-479 120 120 120 197 197 197 F F F 24 480-499 122 122 122 201 201 201 F F F 25 500-519 114 114 114 190 190 190 F F F 26 520-539 126 126 126 206 206 206 F F F 27 540-559 126 126 126 206 206 206 F F F 28 560-579 125 125 125 205 205 205 F F F 29 580-599 124 124 124 204 204 204 F F F Max gen: 127, Max flush: 208 Begin loop 64 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:34.475538042Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1645,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:46:34.482547Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1645,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:34.482598018Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1645,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:34.489354204Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1645,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:34.489618566Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1645} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:34.489653695Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1645} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:34.489662835Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1645} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:34.489674544Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1645} {"msg":"Crucible 71933003-fabe-416e-96c5-c5d8af6b4877 has session id: 2b37df17-6a13-463b-966b-25b82f389e30","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:34.489963566Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1645,"session_id":"2b37df17-6a13-463b-966b-25b82f389e30"} {"msg":"Upstairs opts: Upstairs UUID: 71933003-fabe-416e-96c5-c5d8af6b4877, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:34.489984745Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1645,"session_id":"2b37df17-6a13-463b-966b-25b82f389e30"} {"msg":"Crucible stats registered with UUID: 71933003-fabe-416e-96c5-c5d8af6b4877","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:34.489994715Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1645,"session_id":"2b37df17-6a13-463b-966b-25b82f389e30"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:46:34.490003685Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1645,"session_id":"2b37df17-6a13-463b-966b-25b82f389e30"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:34.490147431Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1645,"":"io task","client":"1","":"downstairs","session_id":"2b37df17-6a13-463b-966b-25b82f389e30"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:34.49016756Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1645,"":"io task","client":"0","":"downstairs","session_id":"2b37df17-6a13-463b-966b-25b82f389e30"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:34.49017766Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1645,"":"io task","client":"2","":"downstairs","session_id":"2b37df17-6a13-463b-966b-25b82f389e30"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:34.490316296Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1645,"task":"crutest"} {"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:34.490334865Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1645} {"msg":"71933003-fabe-416e-96c5-c5d8af6b4877 active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:34.490360615Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1645,"session_id":"2b37df17-6a13-463b-966b-25b82f389e30"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:34.490369814Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1645,"client":"0","":"downstairs","session_id":"2b37df17-6a13-463b-966b-25b82f389e30"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:34.490379564Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1645,"client":"1","":"downstairs","session_id":"2b37df17-6a13-463b-966b-25b82f389e30"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:34.490389194Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1645,"client":"2","":"downstairs","session_id":"2b37df17-6a13-463b-966b-25b82f389e30"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:34.490398783Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1645,"":"io task","client":"2","":"downstairs","session_id":"2b37df17-6a13-463b-966b-25b82f389e30"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:34.490408823Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1645,"":"io task","client":"0","":"downstairs","session_id":"2b37df17-6a13-463b-966b-25b82f389e30"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:34.490418783Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1645,"":"io task","client":"1","":"downstairs","session_id":"2b37df17-6a13-463b-966b-25b82f389e30"} {"msg":"ds_connection connected from Ok(127.0.0.1:35277)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:34.490448692Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1645,"":"io task","client":"2","":"downstairs","session_id":"2b37df17-6a13-463b-966b-25b82f389e30"} {"msg":"ds_connection connected from Ok(127.0.0.1:45579)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:34.490459542Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1645,"":"io task","client":"0","":"downstairs","session_id":"2b37df17-6a13-463b-966b-25b82f389e30"} {"msg":"ds_connection connected from Ok(127.0.0.1:48151)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:34.490469881Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1645,"":"io task","client":"1","":"downstairs","session_id":"2b37df17-6a13-463b-966b-25b82f389e30"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:34.491234349Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1645,"client":"1","":"downstairs","session_id":"2b37df17-6a13-463b-966b-25b82f389e30"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:34.491260649Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1645,"client":"0","":"downstairs","session_id":"2b37df17-6a13-463b-966b-25b82f389e30"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:34.491333856Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1645,"client":"2","":"downstairs","session_id":"2b37df17-6a13-463b-966b-25b82f389e30"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:34.49157296Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1645,"session_id":"2b37df17-6a13-463b-966b-25b82f389e30"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:34.491610799Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1645,"session_id":"2b37df17-6a13-463b-966b-25b82f389e30"} {"msg":"[0]R flush_numbers[0..12]: [204, 204, 204, 206, 206, 202, 205, 208, 196, 206, 206, 207]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:34.491669297Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1645,"":"downstairs","":"downstairs","session_id":"2b37df17-6a13-463b-966b-25b82f389e30"} {"msg":"[0]R generation[0..12]: [124, 124, 124, 126, 126, 123, 125, 127, 118, 126, 126, 126]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:34.491681816Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1645,"":"downstairs","":"downstairs","session_id":"2b37df17-6a13-463b-966b-25b82f389e30"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:34.491693036Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1645,"":"downstairs","":"downstairs","session_id":"2b37df17-6a13-463b-966b-25b82f389e30"} {"msg":"[1]R flush_numbers[0..12]: [204, 204, 204, 206, 206, 202, 205, 208, 196, 206, 206, 207]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:34.491703966Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1645,"":"downstairs","":"downstairs","session_id":"2b37df17-6a13-463b-966b-25b82f389e30"} {"msg":"[1]R generation[0..12]: [124, 124, 124, 126, 126, 123, 125, 127, 118, 126, 126, 126]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:34.491714476Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1645,"":"downstairs","":"downstairs","session_id":"2b37df17-6a13-463b-966b-25b82f389e30"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:34.491725375Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1645,"":"downstairs","":"downstairs","session_id":"2b37df17-6a13-463b-966b-25b82f389e30"} {"msg":"[2]R flush_numbers[0..12]: [204, 204, 204, 206, 206, 202, 205, 208, 196, 206, 206, 207]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:34.491736005Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1645,"":"downstairs","":"downstairs","session_id":"2b37df17-6a13-463b-966b-25b82f389e30"} {"msg":"[2]R generation[0..12]: [124, 124, 124, 126, 126, 123, 125, 127, 118, 126, 126, 126]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:34.491746435Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1645,"":"downstairs","":"downstairs","session_id":"2b37df17-6a13-463b-966b-25b82f389e30"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:34.491757254Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1645,"":"downstairs","":"downstairs","session_id":"2b37df17-6a13-463b-966b-25b82f389e30"} {"msg":"Max found gen is 128","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:34.491766644Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1645,"":"downstairs","":"downstairs","session_id":"2b37df17-6a13-463b-966b-25b82f389e30"} {"msg":"Generation requested: 128 >= found:128","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:34.491776374Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1645,"":"downstairs","":"downstairs","session_id":"2b37df17-6a13-463b-966b-25b82f389e30"} {"msg":"Next flush: 209","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:34.491787943Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1645,"":"downstairs","":"downstairs","session_id":"2b37df17-6a13-463b-966b-25b82f389e30"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:34.491797303Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1645,"":"downstairs","":"downstairs","session_id":"2b37df17-6a13-463b-966b-25b82f389e30"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:34.491807363Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1645,"session_id":"2b37df17-6a13-463b-966b-25b82f389e30"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:34.491816213Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1645,"client":"0","":"downstairs","session_id":"2b37df17-6a13-463b-966b-25b82f389e30"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:34.491826232Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1645,"client":"1","":"downstairs","session_id":"2b37df17-6a13-463b-966b-25b82f389e30"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:34.491836042Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1645,"client":"2","":"downstairs","session_id":"2b37df17-6a13-463b-966b-25b82f389e30"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:34.491845982Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1645,"session_id":"2b37df17-6a13-463b-966b-25b82f389e30"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:34.491855051Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1645,"session_id":"2b37df17-6a13-463b-966b-25b82f389e30"} {"msg":"71933003-fabe-416e-96c5-c5d8af6b4877 is now active with session: 2b37df17-6a13-463b-966b-25b82f389e30","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:34.491864771Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1645,"session_id":"2b37df17-6a13-463b-966b-25b82f389e30"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:34.491873421Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1645,"session_id":"2b37df17-6a13-463b-966b-25b82f389e30"} {"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:34.491882381Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1645} {"msg":"Activated sub_volume 71933003-fabe-416e-96c5-c5d8af6b4877","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:34.49189023Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1645} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:34.49189846Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1645,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:34.492234131Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1645,"task":"crutest"} 01/30 Write block 255 len 32768 data: 12 10 7 7 8 9 9 10 02/30 Write block 152 len 28672 data: 6 5 7 7 8 8 12 03/30 Read block 437 len 32768 04/30 Read block 73 len 20480 05/30 Read block 399 len 8192 06/30 Write block 251 len 40960 data: 7 6 9 8 13 11 8 8 9 10 07/30 Write block 353 len 12288 data: 9 9 10 08/30 Read block 2 len 8192 09/30 Read block 270 len 16384 10/30 Write block 310 len 36864 data: 7 8 8 9 10 9 8 6 5 11/30 Read block 20 len 24576 12/30 Write block 346 len 40960 data: 11 10 10 9 7 5 6 10 10 11 13/30 Read block 181 len 16384 14/30 Write block 505 len 4096 data: 11 15/30 Write block 312 len 16384 data: 9 10 11 10 16/30 Write block 111 len 12288 data: 8 10 10 17/30 Read block 61 len 4096 18/30 Read block 34 len 12288 19/30 Read block 199 len 24576 20/30 Read block 164 len 28672 21/30 Flush 22/30 Write block 143 len 8192 data: 12 12 23/30 Read block 331 len 12288 24/30 Write block 505 len 28672 data: 12 11 12 7 7 9 15 25/30 Read block 529 len 20480 26/30 Read block 422 len 24576 27/30 Read block 413 len 28672 28/30 Read block 268 len 28672 29/30 Write block 390 len 4096 data: 8 30/30 Read block 491 len 32768 ---------------------------------------------------------------- Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Write 7 Sent Done Done false 1002 Acked Read 8 Sent Done Done false 1003 Acked Read 5 Sent Done Done false 1004 Acked Read 2 Sent Done Done false 1005 Acked Write 10 Sent Done Done false 1006 Acked Write 3 Sent Done Done false 1007 Acked Read 2 Sent Done Done false 1008 Acked Read 4 Sent Done Done false 1009 Acked Write 9 Sent Done Done false 1010 Acked Read 6 Sent Done Done false 1011 Acked Write 10 Sent Done Done false 1012 Acked Read 4 Sent Done Done false 1013 Acked Write 1 Sent Done Done false 1014 Acked Write 4 Sent Done Done false 1015 Acked Write 3 Sent Done Done false 1016 Acked Read 1 Sent Done Done false 1017 Acked Read 3 Sent Done Done false 1018 Acked Read 6 Sent Done Done false 1019 Acked Read 7 Sent Done Done false 1020 Acked Flush 0 Sent Done Done false 1021 Acked Write 2 Sent Done Done false 1022 Acked Read 3 Sent Done Done false 1023 Acked Write 7 Sent Done Done false 1024 Acked Read 5 Sent Done Done false 1025 Acked Read 6 Sent Done Done false 1026 Acked Read 7 Sent Done Done false 1027 Acked Read 7 Sent Done Done false 1028 Acked Write 1 Sent Done Done false 1029 Acked Read 8 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 0 0 29 Done 1 30 30 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:46:34.501093625Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1645} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:34.50127176Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1645,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 124 124 204 204 204 F F F 1 020-039 124 124 124 204 204 204 F F F 2 040-059 124 124 124 204 204 204 F F F 3 060-079 126 126 126 206 206 206 F F F 4 080-099 126 126 126 206 206 206 F F F 5 100-119 123 128 128 202 209 209 F F F <--- 6 120-139 125 125 125 205 205 205 F F F 7 140-159 127 128 128 208 209 209 F T T <--- 8 160-179 118 118 118 196 196 196 F F F 9 180-199 126 126 126 206 206 206 F F F 10 200-219 126 126 126 206 206 206 F F F 11 220-239 126 126 126 207 207 207 F F F 12 240-259 115 128 128 191 209 209 T F F <--- 13 260-279 122 128 128 200 209 209 T F F <--- 14 280-299 123 123 123 202 202 202 F F F 15 300-319 127 128 128 208 209 209 F F F <--- 16 320-339 127 127 127 208 208 208 F F F 17 340-359 126 128 128 207 209 209 F F F <--- 18 360-379 124 124 124 204 204 204 F F F 19 380-399 112 112 112 187 187 187 F T T <--- 20 400-419 124 124 124 204 204 204 F F F 21 420-439 127 127 127 208 208 208 F F F 22 440-459 127 127 127 208 208 208 F F F 23 460-479 120 120 120 197 197 197 F F F 24 480-499 122 122 122 201 201 201 F F F 25 500-519 114 128 128 190 209 209 F T T <--- 26 520-539 126 126 126 206 206 206 F F F 27 540-559 126 126 126 206 206 206 F F F 28 560-579 125 125 125 205 205 205 F F F 29 580-599 124 124 124 204 204 204 F F F Max gen: 128, Max flush: 209 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 64 Verifying data now No change after verify Loop: 64 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 124 124 204 204 204 F F F 1 020-039 124 124 124 204 204 204 F F F 2 040-059 124 124 124 204 204 204 F F F 3 060-079 126 126 126 206 206 206 F F F 4 080-099 126 126 126 206 206 206 F F F 5 100-119 128 128 128 209 209 209 F F F 6 120-139 125 125 125 205 205 205 F F F 7 140-159 129 129 129 210 210 210 F F F 8 160-179 118 118 118 196 196 196 F F F 9 180-199 126 126 126 206 206 206 F F F 10 200-219 126 126 126 206 206 206 F F F 11 220-239 126 126 126 207 207 207 F F F 12 240-259 128 128 128 209 209 209 F F F 13 260-279 128 128 128 209 209 209 F F F 14 280-299 123 123 123 202 202 202 F F F 15 300-319 128 128 128 209 209 209 F F F 16 320-339 127 127 127 208 208 208 F F F 17 340-359 128 128 128 209 209 209 F F F 18 360-379 124 124 124 204 204 204 F F F 19 380-399 129 129 129 210 210 210 F F F 20 400-419 124 124 124 204 204 204 F F F 21 420-439 127 127 127 208 208 208 F F F 22 440-459 127 127 127 208 208 208 F F F 23 460-479 120 120 120 197 197 197 F F F 24 480-499 122 122 122 201 201 201 F F F 25 500-519 129 129 129 210 210 210 F F F 26 520-539 126 126 126 206 206 206 F F F 27 540-559 126 126 126 206 206 206 F F F 28 560-579 125 125 125 205 205 205 F F F 29 580-599 124 124 124 204 204 204 F F F Max gen: 129, Max flush: 210 Begin loop 65 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:38.743691939Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1653,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:38.751841794Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1653,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:38.752134635Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1653} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:38.752159165Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1653} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:38.752167624Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1653} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:38.752179074Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1653} {"msg":"Crucible 30181df8-e0f0-4f17-a749-32dea56cee93 has session id: 0484cc90-d7ce-4dd9-94b4-3d77bcb121a6","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:38.752529724Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1653,"session_id":"0484cc90-d7ce-4dd9-94b4-3d77bcb121a6"} {"msg":"Upstairs opts: Upstairs UUID: 30181df8-e0f0-4f17-a749-32dea56cee93, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:38.752558833Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1653,"session_id":"0484cc90-d7ce-4dd9-94b4-3d77bcb121a6"} {"msg":"Crucible stats registered with UUID: 30181df8-e0f0-4f17-a749-32dea56cee93","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:38.752569933Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1653,"session_id":"0484cc90-d7ce-4dd9-94b4-3d77bcb121a6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:46:38.752579052Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1653,"session_id":"0484cc90-d7ce-4dd9-94b4-3d77bcb121a6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:38.752702129Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1653,"":"io task","client":"2","":"downstairs","session_id":"0484cc90-d7ce-4dd9-94b4-3d77bcb121a6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:38.752720738Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1653,"":"io task","client":"0","":"downstairs","session_id":"0484cc90-d7ce-4dd9-94b4-3d77bcb121a6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:38.752730888Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1653,"":"io task","client":"1","":"downstairs","session_id":"0484cc90-d7ce-4dd9-94b4-3d77bcb121a6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:38.752890264Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1653,"task":"crutest"} {"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:38.752908453Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1653} {"msg":"30181df8-e0f0-4f17-a749-32dea56cee93 active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:38.752932202Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1653,"session_id":"0484cc90-d7ce-4dd9-94b4-3d77bcb121a6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:38.752941542Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1653,"client":"0","":"downstairs","session_id":"0484cc90-d7ce-4dd9-94b4-3d77bcb121a6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:38.752953612Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1653,"client":"1","":"downstairs","session_id":"0484cc90-d7ce-4dd9-94b4-3d77bcb121a6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:38.752963431Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1653,"client":"2","":"downstairs","session_id":"0484cc90-d7ce-4dd9-94b4-3d77bcb121a6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:38.752972901Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1653,"":"io task","client":"2","":"downstairs","session_id":"0484cc90-d7ce-4dd9-94b4-3d77bcb121a6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:38.752982831Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1653,"":"io task","client":"0","":"downstairs","session_id":"0484cc90-d7ce-4dd9-94b4-3d77bcb121a6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:38.75301346Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1653,"":"io task","client":"1","":"downstairs","session_id":"0484cc90-d7ce-4dd9-94b4-3d77bcb121a6"} {"msg":"ds_connection connected from Ok(127.0.0.1:36482)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:38.753087658Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1653,"":"io task","client":"2","":"downstairs","session_id":"0484cc90-d7ce-4dd9-94b4-3d77bcb121a6"} {"msg":"ds_connection connected from Ok(127.0.0.1:55593)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:38.753113877Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1653,"":"io task","client":"1","":"downstairs","session_id":"0484cc90-d7ce-4dd9-94b4-3d77bcb121a6"} {"msg":"ds_connection connected from Ok(127.0.0.1:56147)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:38.753158696Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1653,"":"io task","client":"0","":"downstairs","session_id":"0484cc90-d7ce-4dd9-94b4-3d77bcb121a6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:38.753748679Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1653,"client":"0","":"downstairs","session_id":"0484cc90-d7ce-4dd9-94b4-3d77bcb121a6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:38.753762878Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1653,"client":"1","":"downstairs","session_id":"0484cc90-d7ce-4dd9-94b4-3d77bcb121a6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:38.753885175Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1653,"client":"2","":"downstairs","session_id":"0484cc90-d7ce-4dd9-94b4-3d77bcb121a6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:38.754072789Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1653,"session_id":"0484cc90-d7ce-4dd9-94b4-3d77bcb121a6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:38.754082329Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1653,"session_id":"0484cc90-d7ce-4dd9-94b4-3d77bcb121a6"} {"msg":"[0]R flush_numbers[0..12]: [204, 204, 204, 206, 206, 209, 205, 210, 196, 206, 206, 207]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:38.754183626Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1653,"":"downstairs","":"downstairs","session_id":"0484cc90-d7ce-4dd9-94b4-3d77bcb121a6"} {"msg":"[0]R generation[0..12]: [124, 124, 124, 126, 126, 128, 125, 129, 118, 126, 126, 126]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:38.754194786Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1653,"":"downstairs","":"downstairs","session_id":"0484cc90-d7ce-4dd9-94b4-3d77bcb121a6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:38.754208265Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1653,"":"downstairs","":"downstairs","session_id":"0484cc90-d7ce-4dd9-94b4-3d77bcb121a6"} {"msg":"[1]R flush_numbers[0..12]: [204, 204, 204, 206, 206, 209, 205, 210, 196, 206, 206, 207]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:38.754218995Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1653,"":"downstairs","":"downstairs","session_id":"0484cc90-d7ce-4dd9-94b4-3d77bcb121a6"} {"msg":"[1]R generation[0..12]: [124, 124, 124, 126, 126, 128, 125, 129, 118, 126, 126, 126]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:38.754229425Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1653,"":"downstairs","":"downstairs","session_id":"0484cc90-d7ce-4dd9-94b4-3d77bcb121a6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:38.754240105Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1653,"":"downstairs","":"downstairs","session_id":"0484cc90-d7ce-4dd9-94b4-3d77bcb121a6"} {"msg":"[2]R flush_numbers[0..12]: [204, 204, 204, 206, 206, 209, 205, 210, 196, 206, 206, 207]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:38.754250494Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1653,"":"downstairs","":"downstairs","session_id":"0484cc90-d7ce-4dd9-94b4-3d77bcb121a6"} {"msg":"[2]R generation[0..12]: [124, 124, 124, 126, 126, 128, 125, 129, 118, 126, 126, 126]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:38.754260764Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1653,"":"downstairs","":"downstairs","session_id":"0484cc90-d7ce-4dd9-94b4-3d77bcb121a6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:38.754271534Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1653,"":"downstairs","":"downstairs","session_id":"0484cc90-d7ce-4dd9-94b4-3d77bcb121a6"} {"msg":"Max found gen is 130","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:38.754280883Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1653,"":"downstairs","":"downstairs","session_id":"0484cc90-d7ce-4dd9-94b4-3d77bcb121a6"} {"msg":"Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:38.754290473Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1653,"":"downstairs","":"downstairs","session_id":"0484cc90-d7ce-4dd9-94b4-3d77bcb121a6"} {"msg":"Next flush: 211","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:38.754299693Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1653,"":"downstairs","":"downstairs","session_id":"0484cc90-d7ce-4dd9-94b4-3d77bcb121a6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:38.754309123Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1653,"":"downstairs","":"downstairs","session_id":"0484cc90-d7ce-4dd9-94b4-3d77bcb121a6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:38.754319102Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1653,"session_id":"0484cc90-d7ce-4dd9-94b4-3d77bcb121a6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:38.754327932Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1653,"client":"0","":"downstairs","session_id":"0484cc90-d7ce-4dd9-94b4-3d77bcb121a6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:38.754337712Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1653,"client":"1","":"downstairs","session_id":"0484cc90-d7ce-4dd9-94b4-3d77bcb121a6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:38.754347461Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1653,"client":"2","":"downstairs","session_id":"0484cc90-d7ce-4dd9-94b4-3d77bcb121a6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:38.754358821Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1653,"session_id":"0484cc90-d7ce-4dd9-94b4-3d77bcb121a6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:38.754368131Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1653,"session_id":"0484cc90-d7ce-4dd9-94b4-3d77bcb121a6"} {"msg":"30181df8-e0f0-4f17-a749-32dea56cee93 is now active with session: 0484cc90-d7ce-4dd9-94b4-3d77bcb121a6","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:38.754378081Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1653,"session_id":"0484cc90-d7ce-4dd9-94b4-3d77bcb121a6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:38.75438672Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1653,"session_id":"0484cc90-d7ce-4dd9-94b4-3d77bcb121a6"} {"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:38.75439589Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1653} {"msg":"Activated sub_volume 30181df8-e0f0-4f17-a749-32dea56cee93","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:38.75440389Z","Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1653} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:38.754417079Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1653,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:38.754678752Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1653,"task":"crutest"} 01/30 Write block 136 len 4096 data: 13 02/30 Read block 378 len 24576 03/30 Write block 237 len 12288 data: 7 6 7 04/30 Write block 234 len 32768 data: 11 8 7 8 7 8 8 9 05/30 Write block 70 len 36864 data: 10 11 11 11 11 11 10 10 12 06/30 Read block 32 len 8192 07/30 Write block 100 len 20480 data: 9 5 5 5 5 08/30 Write block 542 len 4096 data: 13 09/30 Write block 128 len 28672 data: 12 10 11 11 10 11 11 10/30 Read block 475 len 36864 11/30 Write block 394 len 36864 data: 8 11 12 9 7 10 12 12 14 12/30 Read block 326 len 28672 13/30 Read block 280 len 20480 14/30 Write block 378 len 36864 data: 14 14 13 13 16 12 11 11 9 15/30 Write block 293 len 12288 data: 13 13 13 16/30 Write block 86 len 16384 data: 9 8 5 6 17/30 Write block 326 len 12288 data: 14 15 12 18/30 Read block 434 len 32768 19/30 Write block 309 len 16384 data: 8 8 9 10 20/30 Read block 295 len 8192 21/30 Read block 433 len 36864 22/30 Write block 416 len 36864 data: 11 11 10 11 11 11 10 11 12 23/30 Write block 213 len 16384 data: 16 13 10 11 24/30 Read block 595 len 8192 25/30 Write block 149 len 24576 data: 6 6 7 7 6 8 26/30 Flush 27/30 Write block 335 len 12288 data: 11 12 13 28/30 Write block 142 len 24576 data: 13 13 13 11 12 11 29/30 Read block 109 len 4096 30/30 Write block 95 len 36864 data: 11 12 12 11 10 10 6 6 6 ---------------------------------------------------------------- Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Sent false 1001 Acked Read 6 Done Done Sent false 1002 Acked Write 3 Done Done Sent false 1003 Acked Write 8 Done Done Sent false 1004 Acked Write 9 Done Done Sent false 1005 Acked Read 2 Done Done Sent false 1006 Acked Write 5 Done Done Sent false 1007 Acked Write 1 Done Done Sent false 1008 Acked Write 7 Done Done Sent false 1009 Acked Read 9 Done Done Sent false 1010 Acked Write 9 Done Done Sent false 1011 Acked Read 7 Done Done Sent false 1012 Acked Read 5 Done Done Sent false 1013 Acked Write 9 Done Done Sent false 1014 Acked Write 3 Done Done Sent false 1015 Acked Write 4 Done Done Sent false 1016 Acked Write 3 Done Done Sent false 1017 Acked Read 8 Done Done Sent false 1018 Acked Write 4 Done Done Sent false 1019 Acked Read 2 Done Done Sent false 1020 Acked Read 9 Done Done Sent false 1021 Acked Write 9 Done Done Sent false 1022 Acked Write 4 Done Done Sent false 1023 Acked Read 2 Done Done Sent false 1024 Acked Write 6 Done Done Sent false 1025 Acked Flush 0 Done Done Sent false 1026 Acked Write 3 Done Done Sent false 1027 Acked Write 6 Done Done Sent false 1028 Acked Read 1 Done Done Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 30 32 Done 29 29 0 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:46:38.762917114Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1653} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:38.763100279Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1653,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 124 124 204 204 204 F F F 1 020-039 124 124 124 204 204 204 F F F 2 040-059 124 124 124 204 204 204 F F F 3 060-079 130 130 126 211 211 206 F F F <--- 4 080-099 130 130 126 211 211 206 T T F <--- 5 100-119 130 130 128 211 211 209 T T F <--- 6 120-139 130 130 125 211 211 205 F F F <--- 7 140-159 130 130 129 211 211 210 T T F <--- 8 160-179 118 118 118 196 196 196 F F F 9 180-199 126 126 126 206 206 206 F F F 10 200-219 130 130 126 211 211 206 F F F <--- 11 220-239 130 130 126 211 211 207 F F F <--- 12 240-259 130 130 128 211 211 209 F F F <--- 13 260-279 128 128 128 209 209 209 F F F 14 280-299 130 130 123 211 211 202 F F F <--- 15 300-319 130 130 128 211 211 209 F F F <--- 16 320-339 130 130 127 211 211 208 T T F <--- 17 340-359 128 128 128 209 209 209 F F F 18 360-379 130 130 124 211 211 204 F F F <--- 19 380-399 130 130 129 211 211 210 F F F <--- 20 400-419 130 130 124 211 211 204 F F F <--- 21 420-439 130 130 127 211 211 208 F F F <--- 22 440-459 127 127 127 208 208 208 F F F 23 460-479 120 120 120 197 197 197 F F F 24 480-499 122 122 122 201 201 201 F F F 25 500-519 129 129 129 210 210 210 F F F 26 520-539 126 126 126 206 206 206 F F F 27 540-559 130 130 126 211 211 206 F F F <--- 28 560-579 125 125 125 205 205 205 F F F 29 580-599 124 124 124 204 204 204 F F F Max gen: 130, Max flush: 211 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 65 Verifying data now No change after verify Loop: 65 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 124 124 204 204 204 F F F 1 020-039 124 124 124 204 204 204 F F F 2 040-059 124 124 124 204 204 204 F F F 3 060-079 130 130 130 211 211 211 F F F 4 080-099 131 131 131 212 212 212 F F F 5 100-119 131 131 131 212 212 212 F F F 6 120-139 130 130 130 211 211 211 F F F 7 140-159 131 131 131 212 212 212 F F F 8 160-179 118 118 118 196 196 196 F F F 9 180-199 126 126 126 206 206 206 F F F 10 200-219 130 130 130 211 211 211 F F F 11 220-239 130 130 130 211 211 211 F F F 12 240-259 130 130 130 211 211 211 F F F 13 260-279 128 128 128 209 209 209 F F F 14 280-299 130 130 130 211 211 211 F F F 15 300-319 130 130 130 211 211 211 F F F 16 320-339 131 131 131 212 212 212 F F F 17 340-359 128 128 128 209 209 209 F F F 18 360-379 130 130 130 211 211 211 F F F 19 380-399 130 130 130 211 211 211 F F F 20 400-419 130 130 130 211 211 211 F F F 21 420-439 130 130 130 211 211 211 F F F 22 440-459 127 127 127 208 208 208 F F F 23 460-479 120 120 120 197 197 197 F F F 24 480-499 122 122 122 201 201 201 F F F 25 500-519 129 129 129 210 210 210 F F F 26 520-539 126 126 126 206 206 206 F F F 27 540-559 130 130 130 211 211 211 F F F 28 560-579 125 125 125 205 205 205 F F F 29 580-599 124 124 124 204 204 204 F F F Max gen: 131, Max flush: 212 Begin loop 66 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:43.013262194Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1661,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:43.021287583Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1661,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:43.021605664Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1661} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:43.021634113Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1661} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:43.021642762Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1661} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:43.021654302Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1661} {"msg":"Crucible d544a7ad-2731-451a-b258-6bb57420393d has session id: 9d235522-4445-480d-9e55-3ffb7b5c73d3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:43.022008522Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1661,"session_id":"9d235522-4445-480d-9e55-3ffb7b5c73d3"} {"msg":"Upstairs opts: Upstairs UUID: d544a7ad-2731-451a-b258-6bb57420393d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:43.022037371Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1661,"session_id":"9d235522-4445-480d-9e55-3ffb7b5c73d3"} {"msg":"Crucible stats registered with UUID: d544a7ad-2731-451a-b258-6bb57420393d","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:43.022048121Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1661,"session_id":"9d235522-4445-480d-9e55-3ffb7b5c73d3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:46:43.02205716Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1661,"session_id":"9d235522-4445-480d-9e55-3ffb7b5c73d3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:43.022187297Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1661,"":"io task","client":"1","":"downstairs","session_id":"9d235522-4445-480d-9e55-3ffb7b5c73d3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:43.022219396Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1661,"":"io task","client":"0","":"downstairs","session_id":"9d235522-4445-480d-9e55-3ffb7b5c73d3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:43.022230065Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1661,"":"io task","client":"2","":"downstairs","session_id":"9d235522-4445-480d-9e55-3ffb7b5c73d3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:43.022370351Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1661,"task":"crutest"} {"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:43.022389411Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1661} {"msg":"d544a7ad-2731-451a-b258-6bb57420393d active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:43.02241509Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1661,"session_id":"9d235522-4445-480d-9e55-3ffb7b5c73d3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:43.02242473Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1661,"client":"0","":"downstairs","session_id":"9d235522-4445-480d-9e55-3ffb7b5c73d3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:43.02243442Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1661,"client":"1","":"downstairs","session_id":"9d235522-4445-480d-9e55-3ffb7b5c73d3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:43.022443859Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1661,"client":"2","":"downstairs","session_id":"9d235522-4445-480d-9e55-3ffb7b5c73d3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:43.022453369Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1661,"":"io task","client":"2","":"downstairs","session_id":"9d235522-4445-480d-9e55-3ffb7b5c73d3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:43.022483278Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1661,"":"io task","client":"0","":"downstairs","session_id":"9d235522-4445-480d-9e55-3ffb7b5c73d3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:43.022493408Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1661,"":"io task","client":"1","":"downstairs","session_id":"9d235522-4445-480d-9e55-3ffb7b5c73d3"} {"msg":"ds_connection connected from Ok(127.0.0.1:57526)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:43.022516017Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1661,"":"io task","client":"2","":"downstairs","session_id":"9d235522-4445-480d-9e55-3ffb7b5c73d3"} {"msg":"ds_connection connected from Ok(127.0.0.1:46759)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:43.022533597Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1661,"":"io task","client":"1","":"downstairs","session_id":"9d235522-4445-480d-9e55-3ffb7b5c73d3"} {"msg":"ds_connection connected from Ok(127.0.0.1:60958)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:43.022546536Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1661,"":"io task","client":"0","":"downstairs","session_id":"9d235522-4445-480d-9e55-3ffb7b5c73d3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:43.023194748Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1661,"client":"1","":"downstairs","session_id":"9d235522-4445-480d-9e55-3ffb7b5c73d3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:43.023218087Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1661,"client":"0","":"downstairs","session_id":"9d235522-4445-480d-9e55-3ffb7b5c73d3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:43.023330704Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1661,"client":"2","":"downstairs","session_id":"9d235522-4445-480d-9e55-3ffb7b5c73d3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:43.023549117Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1661,"session_id":"9d235522-4445-480d-9e55-3ffb7b5c73d3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:43.023595416Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1661,"session_id":"9d235522-4445-480d-9e55-3ffb7b5c73d3"} {"msg":"[0]R flush_numbers[0..12]: [204, 204, 204, 211, 212, 212, 211, 212, 196, 206, 211, 211]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:43.023635215Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1661,"":"downstairs","":"downstairs","session_id":"9d235522-4445-480d-9e55-3ffb7b5c73d3"} {"msg":"[0]R generation[0..12]: [124, 124, 124, 130, 131, 131, 130, 131, 118, 126, 130, 130]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:43.023646535Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1661,"":"downstairs","":"downstairs","session_id":"9d235522-4445-480d-9e55-3ffb7b5c73d3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:43.023657564Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1661,"":"downstairs","":"downstairs","session_id":"9d235522-4445-480d-9e55-3ffb7b5c73d3"} {"msg":"[1]R flush_numbers[0..12]: [204, 204, 204, 211, 212, 212, 211, 212, 196, 206, 211, 211]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:43.023668454Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1661,"":"downstairs","":"downstairs","session_id":"9d235522-4445-480d-9e55-3ffb7b5c73d3"} {"msg":"[1]R generation[0..12]: [124, 124, 124, 130, 131, 131, 130, 131, 118, 126, 130, 130]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:43.023678854Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1661,"":"downstairs","":"downstairs","session_id":"9d235522-4445-480d-9e55-3ffb7b5c73d3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:43.023689643Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1661,"":"downstairs","":"downstairs","session_id":"9d235522-4445-480d-9e55-3ffb7b5c73d3"} {"msg":"[2]R flush_numbers[0..12]: [204, 204, 204, 211, 212, 212, 211, 212, 196, 206, 211, 211]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:43.023702683Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1661,"":"downstairs","":"downstairs","session_id":"9d235522-4445-480d-9e55-3ffb7b5c73d3"} {"msg":"[2]R generation[0..12]: [124, 124, 124, 130, 131, 131, 130, 131, 118, 126, 130, 130]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:43.023713173Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1661,"":"downstairs","":"downstairs","session_id":"9d235522-4445-480d-9e55-3ffb7b5c73d3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:43.023724192Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1661,"":"downstairs","":"downstairs","session_id":"9d235522-4445-480d-9e55-3ffb7b5c73d3"} {"msg":"Max found gen is 132","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:43.023733642Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1661,"":"downstairs","":"downstairs","session_id":"9d235522-4445-480d-9e55-3ffb7b5c73d3"} {"msg":"Generation requested: 132 >= found:132","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:43.023743232Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1661,"":"downstairs","":"downstairs","session_id":"9d235522-4445-480d-9e55-3ffb7b5c73d3"} {"msg":"Next flush: 213","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:43.023752592Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1661,"":"downstairs","":"downstairs","session_id":"9d235522-4445-480d-9e55-3ffb7b5c73d3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:43.023761911Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1661,"":"downstairs","":"downstairs","session_id":"9d235522-4445-480d-9e55-3ffb7b5c73d3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:43.023771471Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1661,"session_id":"9d235522-4445-480d-9e55-3ffb7b5c73d3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:43.023780301Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1661,"client":"0","":"downstairs","session_id":"9d235522-4445-480d-9e55-3ffb7b5c73d3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:43.02378997Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1661,"client":"1","":"downstairs","session_id":"9d235522-4445-480d-9e55-3ffb7b5c73d3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:43.02379985Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1661,"client":"2","":"downstairs","session_id":"9d235522-4445-480d-9e55-3ffb7b5c73d3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:43.02380966Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1661,"session_id":"9d235522-4445-480d-9e55-3ffb7b5c73d3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:43.0238187Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1661,"session_id":"9d235522-4445-480d-9e55-3ffb7b5c73d3"} {"msg":"d544a7ad-2731-451a-b258-6bb57420393d is now active with session: 9d235522-4445-480d-9e55-3ffb7b5c73d3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:43.023829049Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1661,"session_id":"9d235522-4445-480d-9e55-3ffb7b5c73d3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:43.023837759Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1661,"session_id":"9d235522-4445-480d-9e55-3ffb7b5c73d3"} {"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:43.023848019Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1661} {"msg":"Activated sub_volume d544a7ad-2731-451a-b258-6bb57420393d","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:43.023856009Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1661} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:43.023864118Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1661,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:43.024202379Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1661,"task":"crutest"} 01/30 Write block 552 len 36864 data: 10 11 8 7 6 9 12 11 11 02/30 Flush 03/30 Write block 248 len 32768 data: 11 9 10 8 7 10 9 14 04/30 Write block 321 len 32768 data: 9 9 12 15 14 15 16 13 05/30 Read block 33 len 4096 06/30 Flush 07/30 Write block 595 len 16384 data: 7 6 5 4 08/30 Write block 461 len 24576 data: 6 5 6 5 3 4 09/30 Write block 380 len 20480 data: 14 14 17 13 12 10/30 Read block 77 len 12288 11/30 Read block 298 len 16384 12/30 Write block 286 len 32768 data: 8 8 8 10 10 10 11 14 13/30 Read block 67 len 24576 14/30 Read block 347 len 40960 15/30 Read block 75 len 40960 16/30 Flush 17/30 Flush 18/30 Write block 377 len 24576 data: 14 15 15 15 15 18 19/30 Write block 18 len 40960 data: 10 7 7 6 11 8 7 7 4 5 20/30 Write block 297 len 8192 data: 14 12 21/30 Write block 361 len 20480 data: 11 14 15 16 13 22/30 Write block 374 len 36864 data: 5 9 14 15 16 16 16 16 19 23/30 Read block 439 len 36864 24/30 Write block 568 len 32768 data: 8 9 9 11 9 11 11 11 25/30 Read block 563 len 20480 26/30 Write block 515 len 40960 data: 7 7 8 10 12 11 14 14 12 11 27/30 Write block 257 len 8192 data: 9 9 28/30 Write block 73 len 32768 data: 12 12 12 11 11 13 11 8 29/30 Read block 165 len 8192 30/30 Write block 266 len 24576 data: 8 9 8 7 7 10 ---------------------------------------------------------------- Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 4 Done Done Done false 1007 Acked Write 6 Done Done Done false 1008 Acked Write 5 Done Done Sent false 1009 Acked Read 3 Done Done Sent false 1010 Acked Read 4 Done Done Sent false 1011 Acked Write 8 Done Done Sent false 1012 Acked Read 6 Done Done Sent false 1013 Acked Read 10 Done Done Sent false 1014 Acked Read 10 Done Done Sent false 1015 Acked Flush 0 Done Done Sent false 1016 Acked Flush 0 Done Done Sent false 1017 Acked Write 6 Done Done Sent false 1018 Acked Write 10 Done Done Sent false 1019 Acked Write 2 Done Done Sent false 1020 Acked Write 5 Done Done Sent false 1021 Acked Write 9 Done Done Sent false 1022 Acked Read 9 Done Done Sent false 1023 Acked Write 8 Done Done Sent false 1024 Acked Read 5 Done Done Sent false 1025 Acked Write 10 Done Done Sent false 1026 Acked Write 2 Done Done Sent false 1027 Acked Write 8 Done Done Sent false 1028 Acked Read 2 Done Done Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 22 24 Done 23 23 2 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) Some(JobId(1005)) Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:46:43.033004915Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1661} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:43.03317997Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1661,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 124 124 204 204 204 T T F <--- 1 020-039 124 124 124 204 204 204 T T F <--- 2 040-059 124 124 124 204 204 204 F F F 3 060-079 130 130 130 211 211 211 T T F <--- 4 080-099 131 131 131 212 212 212 T T F <--- 5 100-119 131 131 131 212 212 212 F F F 6 120-139 130 130 130 211 211 211 F F F 7 140-159 131 131 131 212 212 212 F F F 8 160-179 118 118 118 196 196 196 F F F 9 180-199 126 126 126 206 206 206 F F F 10 200-219 130 130 130 211 211 211 F F F 11 220-239 130 130 130 211 211 211 F F F 12 240-259 132 132 132 214 214 214 T T F <--- 13 260-279 128 128 128 209 209 209 T T F <--- 14 280-299 132 132 130 215 215 211 T T F <--- 15 300-319 130 130 130 211 211 211 F F F 16 320-339 132 132 132 214 214 214 F F F 17 340-359 128 128 128 209 209 209 F F F 18 360-379 130 130 130 211 211 211 T T F <--- 19 380-399 132 132 130 215 215 211 T T F <--- 20 400-419 130 130 130 211 211 211 F F F 21 420-439 130 130 130 211 211 211 F F F 22 440-459 127 127 127 208 208 208 F F F 23 460-479 132 132 120 215 215 197 F F T <--- 24 480-499 122 122 122 201 201 201 F F F 25 500-519 129 129 129 210 210 210 T T F <--- 26 520-539 126 126 126 206 206 206 T T F <--- 27 540-559 132 132 132 213 213 213 F F F 28 560-579 132 132 132 213 213 213 T T F <--- 29 580-599 132 132 124 215 215 204 F F T <--- Max gen: 132, Max flush: 215 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 66 Verifying data now No change after verify Loop: 66 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 133 133 133 216 216 216 F F F 1 020-039 133 133 133 216 216 216 F F F 2 040-059 124 124 124 204 204 204 F F F 3 060-079 133 133 133 216 216 216 F F F 4 080-099 133 133 133 216 216 216 F F F 5 100-119 131 131 131 212 212 212 F F F 6 120-139 130 130 130 211 211 211 F F F 7 140-159 131 131 131 212 212 212 F F F 8 160-179 118 118 118 196 196 196 F F F 9 180-199 126 126 126 206 206 206 F F F 10 200-219 130 130 130 211 211 211 F F F 11 220-239 130 130 130 211 211 211 F F F 12 240-259 133 133 133 216 216 216 F F F 13 260-279 133 133 133 216 216 216 F F F 14 280-299 133 133 133 216 216 216 F F F 15 300-319 130 130 130 211 211 211 F F F 16 320-339 132 132 132 214 214 214 F F F 17 340-359 128 128 128 209 209 209 F F F 18 360-379 133 133 133 216 216 216 F F F 19 380-399 133 133 133 216 216 216 F F F 20 400-419 130 130 130 211 211 211 F F F 21 420-439 130 130 130 211 211 211 F F F 22 440-459 127 127 127 208 208 208 F F F 23 460-479 132 132 132 215 215 215 F F F 24 480-499 122 122 122 201 201 201 F F F 25 500-519 133 133 133 216 216 216 F F F 26 520-539 133 133 133 216 216 216 F F F 27 540-559 132 132 132 213 213 213 F F F 28 560-579 133 133 133 216 216 216 F F F 29 580-599 132 132 132 215 215 215 F F F Max gen: 133, Max flush: 216 Begin loop 67 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:47.283616417Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1669,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:46:47.290573737Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1669,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:47.290627815Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1669,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:47.297296473Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1669,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:47.297559355Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1669} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:47.297594954Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1669} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:47.297604144Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1669} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:47.297615794Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1669} {"msg":"Crucible 51ac1520-dfd2-40b6-b181-f5e498ce191f has session id: 1248252d-d111-44e8-87ac-704adc183c6a","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:47.297963544Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1669,"session_id":"1248252d-d111-44e8-87ac-704adc183c6a"} {"msg":"Upstairs opts: Upstairs UUID: 51ac1520-dfd2-40b6-b181-f5e498ce191f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:47.297987023Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1669,"session_id":"1248252d-d111-44e8-87ac-704adc183c6a"} {"msg":"Crucible stats registered with UUID: 51ac1520-dfd2-40b6-b181-f5e498ce191f","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:47.297997003Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1669,"session_id":"1248252d-d111-44e8-87ac-704adc183c6a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:46:47.298008242Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1669,"session_id":"1248252d-d111-44e8-87ac-704adc183c6a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:47.298161978Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1669,"":"io task","client":"1","":"downstairs","session_id":"1248252d-d111-44e8-87ac-704adc183c6a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:47.298179417Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1669,"":"io task","client":"2","":"downstairs","session_id":"1248252d-d111-44e8-87ac-704adc183c6a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:47.298189647Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1669,"":"io task","client":"0","":"downstairs","session_id":"1248252d-d111-44e8-87ac-704adc183c6a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:47.298310814Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1669,"task":"crutest"} {"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:47.298327963Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1669} {"msg":"51ac1520-dfd2-40b6-b181-f5e498ce191f active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:47.298360092Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1669,"session_id":"1248252d-d111-44e8-87ac-704adc183c6a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:47.298369612Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1669,"client":"0","":"downstairs","session_id":"1248252d-d111-44e8-87ac-704adc183c6a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:47.298379122Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1669,"client":"1","":"downstairs","session_id":"1248252d-d111-44e8-87ac-704adc183c6a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:47.298388631Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1669,"client":"2","":"downstairs","session_id":"1248252d-d111-44e8-87ac-704adc183c6a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:47.298398051Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1669,"":"io task","client":"2","":"downstairs","session_id":"1248252d-d111-44e8-87ac-704adc183c6a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:47.29842214Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1669,"":"io task","client":"0","":"downstairs","session_id":"1248252d-d111-44e8-87ac-704adc183c6a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:47.29843835Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1669,"":"io task","client":"1","":"downstairs","session_id":"1248252d-d111-44e8-87ac-704adc183c6a"} {"msg":"ds_connection connected from Ok(127.0.0.1:64545)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:47.298517328Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1669,"":"io task","client":"2","":"downstairs","session_id":"1248252d-d111-44e8-87ac-704adc183c6a"} {"msg":"ds_connection connected from Ok(127.0.0.1:43354)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:47.298542377Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1669,"":"io task","client":"1","":"downstairs","session_id":"1248252d-d111-44e8-87ac-704adc183c6a"} {"msg":"ds_connection connected from Ok(127.0.0.1:40877)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:47.298553547Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1669,"":"io task","client":"0","":"downstairs","session_id":"1248252d-d111-44e8-87ac-704adc183c6a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:47.299184438Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1669,"client":"2","":"downstairs","session_id":"1248252d-d111-44e8-87ac-704adc183c6a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:47.299222327Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1669,"client":"1","":"downstairs","session_id":"1248252d-d111-44e8-87ac-704adc183c6a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:47.299243317Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1669,"client":"0","":"downstairs","session_id":"1248252d-d111-44e8-87ac-704adc183c6a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:47.29946167Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1669,"session_id":"1248252d-d111-44e8-87ac-704adc183c6a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:47.299511909Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1669,"session_id":"1248252d-d111-44e8-87ac-704adc183c6a"} {"msg":"[0]R flush_numbers[0..12]: [216, 216, 204, 216, 216, 212, 211, 212, 196, 206, 211, 211]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:47.299561817Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1669,"":"downstairs","":"downstairs","session_id":"1248252d-d111-44e8-87ac-704adc183c6a"} {"msg":"[0]R generation[0..12]: [133, 133, 124, 133, 133, 131, 130, 131, 118, 126, 130, 130]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:47.299572957Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1669,"":"downstairs","":"downstairs","session_id":"1248252d-d111-44e8-87ac-704adc183c6a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:47.299583787Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1669,"":"downstairs","":"downstairs","session_id":"1248252d-d111-44e8-87ac-704adc183c6a"} {"msg":"[1]R flush_numbers[0..12]: [216, 216, 204, 216, 216, 212, 211, 212, 196, 206, 211, 211]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:47.299594457Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1669,"":"downstairs","":"downstairs","session_id":"1248252d-d111-44e8-87ac-704adc183c6a"} {"msg":"[1]R generation[0..12]: [133, 133, 124, 133, 133, 131, 130, 131, 118, 126, 130, 130]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:47.299604876Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1669,"":"downstairs","":"downstairs","session_id":"1248252d-d111-44e8-87ac-704adc183c6a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:47.299615706Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1669,"":"downstairs","":"downstairs","session_id":"1248252d-d111-44e8-87ac-704adc183c6a"} {"msg":"[2]R flush_numbers[0..12]: [216, 216, 204, 216, 216, 212, 211, 212, 196, 206, 211, 211]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:47.299626246Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1669,"":"downstairs","":"downstairs","session_id":"1248252d-d111-44e8-87ac-704adc183c6a"} {"msg":"[2]R generation[0..12]: [133, 133, 124, 133, 133, 131, 130, 131, 118, 126, 130, 130]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:47.299636795Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1669,"":"downstairs","":"downstairs","session_id":"1248252d-d111-44e8-87ac-704adc183c6a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:47.299647495Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1669,"":"downstairs","":"downstairs","session_id":"1248252d-d111-44e8-87ac-704adc183c6a"} {"msg":"Max found gen is 134","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:47.299658805Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1669,"":"downstairs","":"downstairs","session_id":"1248252d-d111-44e8-87ac-704adc183c6a"} {"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:47.299668484Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1669,"":"downstairs","":"downstairs","session_id":"1248252d-d111-44e8-87ac-704adc183c6a"} {"msg":"Next flush: 217","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:47.299677764Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1669,"":"downstairs","":"downstairs","session_id":"1248252d-d111-44e8-87ac-704adc183c6a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:47.299687084Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1669,"":"downstairs","":"downstairs","session_id":"1248252d-d111-44e8-87ac-704adc183c6a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:47.299696684Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1669,"session_id":"1248252d-d111-44e8-87ac-704adc183c6a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:47.299705473Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1669,"client":"0","":"downstairs","session_id":"1248252d-d111-44e8-87ac-704adc183c6a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:47.299715213Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1669,"client":"1","":"downstairs","session_id":"1248252d-d111-44e8-87ac-704adc183c6a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:47.299724803Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1669,"client":"2","":"downstairs","session_id":"1248252d-d111-44e8-87ac-704adc183c6a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:47.299734682Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1669,"session_id":"1248252d-d111-44e8-87ac-704adc183c6a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:47.299743732Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1669,"session_id":"1248252d-d111-44e8-87ac-704adc183c6a"} {"msg":"51ac1520-dfd2-40b6-b181-f5e498ce191f is now active with session: 1248252d-d111-44e8-87ac-704adc183c6a","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:47.299753692Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1669,"session_id":"1248252d-d111-44e8-87ac-704adc183c6a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:47.299762452Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1669,"session_id":"1248252d-d111-44e8-87ac-704adc183c6a"} {"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:47.299771551Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1669} {"msg":"Activated sub_volume 51ac1520-dfd2-40b6-b181-f5e498ce191f","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:47.299779411Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1669} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:47.299787391Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1669,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:47.300087222Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1669,"task":"crutest"} 01/30 Write block 285 len 16384 data: 11 9 9 9 02/30 Write block 130 len 12288 data: 12 12 11 03/30 Write block 27 len 16384 data: 6 5 7 8 04/30 Flush 05/30 Write block 523 len 4096 data: 13 06/30 Write block 206 len 4096 data: 13 07/30 Read block 109 len 12288 08/30 Read block 383 len 20480 09/30 Write block 411 len 40960 data: 13 13 13 10 7 12 12 11 12 12 10/30 Write block 444 len 40960 data: 8 7 7 6 6 7 9 7 6 6 11/30 Write block 244 len 28672 data: 8 8 10 12 12 10 11 12/30 Read block 554 len 12288 13/30 Read block 416 len 16384 14/30 Read block 488 len 8192 15/30 Read block 188 len 32768 16/30 Flush 17/30 Write block 142 len 40960 data: 14 14 14 12 13 12 10 7 7 8 18/30 Write block 55 len 16384 data: 10 10 10 13 19/30 Read block 555 len 12288 20/30 Write block 241 len 36864 data: 10 8 8 9 9 11 13 13 11 21/30 Flush 22/30 Write block 256 len 20480 data: 12 10 10 10 11 23/30 Write block 286 len 4096 data: 10 24/30 Write block 587 len 28672 data: 17 15 9 9 8 7 7 25/30 Read block 538 len 36864 26/30 Write block 479 len 40960 data: 12 10 13 9 8 8 11 12 13 14 27/30 Read block 500 len 28672 28/30 Read block 118 len 40960 29/30 Write block 399 len 4096 data: 11 30/30 Write block 355 len 8192 data: 12 10 ---------------------------------------------------------------- Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Write 3 Sent Done Done false 1002 Acked Write 4 Sent Done Done false 1003 Acked Flush 0 Sent Done Done false 1004 Acked Write 1 Sent Done Done false 1005 Acked Write 1 Sent Done Done false 1006 Acked Read 3 Sent Done Done false 1007 Acked Read 5 Sent Done Done false 1008 Acked Write 10 Sent Done Done false 1009 Acked Write 10 Sent Done Done false 1010 Acked Write 7 Sent Done Done false 1011 Acked Read 3 Sent Done Done false 1012 Acked Read 4 Sent Done Done false 1013 Acked Read 2 Sent Done Done false 1014 Acked Read 8 Sent Done Done false 1015 Acked Flush 0 Sent Done Done false 1016 Acked Write 10 Sent Done Done false 1017 Acked Write 4 Sent Done Done false 1018 Acked Read 3 Sent Done Done false 1019 Acked Write 9 Sent Done Done false 1020 Acked Flush 0 Sent Done Done false 1021 Acked Write 5 Sent Done Done false 1022 Acked Write 1 Sent Done Done false 1023 Acked Write 7 Sent Done Done false 1024 Acked Read 9 Sent Done Done false 1025 Acked Write 10 Sent Done Done false 1026 Acked Read 7 Sent Done Done false 1027 Acked Read 10 Sent Done Done false 1028 Acked Write 1 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 2 2 33 Done 1 28 28 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:46:47.30848192Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1669} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:47.308659505Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1669,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 133 133 133 216 216 216 F F F 1 020-039 133 134 134 216 217 217 F F F <--- 2 040-059 124 134 134 204 219 219 F F F <--- 3 060-079 133 133 133 216 216 216 F F F 4 080-099 133 133 133 216 216 216 F F F 5 100-119 131 131 131 212 212 212 F F F 6 120-139 130 134 134 211 217 217 F F F <--- 7 140-159 131 134 134 212 219 219 F F F <--- 8 160-179 118 118 118 196 196 196 F F F 9 180-199 126 126 126 206 206 206 F F F 10 200-219 130 134 134 211 218 218 F F F <--- 11 220-239 130 130 130 211 211 211 F F F 12 240-259 133 134 134 216 219 219 F T T <--- 13 260-279 133 133 133 216 216 216 F T T <--- 14 280-299 133 134 134 216 217 217 T T T <--- 15 300-319 130 130 130 211 211 211 F F F 16 320-339 132 132 132 214 214 214 F F F 17 340-359 128 128 128 209 209 209 F T T <--- 18 360-379 133 133 133 216 216 216 F F F 19 380-399 133 133 133 216 216 216 F T T <--- 20 400-419 130 134 134 211 218 218 F F F <--- 21 420-439 130 134 134 211 218 218 F F F <--- 22 440-459 127 134 134 208 218 218 F F F <--- 23 460-479 132 132 132 215 215 215 F T T <--- 24 480-499 122 122 122 201 201 201 F T T <--- 25 500-519 133 133 133 216 216 216 F F F 26 520-539 133 134 134 216 218 218 F F F <--- 27 540-559 132 132 132 213 213 213 F F F 28 560-579 133 133 133 216 216 216 F F F 29 580-599 132 132 132 215 215 215 F T T <--- Max gen: 134, Max flush: 219 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 67 Verifying data now No change after verify Loop: 67 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 133 133 133 216 216 216 F F F 1 020-039 134 134 134 217 217 217 F F F 2 040-059 134 134 134 219 219 219 F F F 3 060-079 133 133 133 216 216 216 F F F 4 080-099 133 133 133 216 216 216 F F F 5 100-119 131 131 131 212 212 212 F F F 6 120-139 134 134 134 217 217 217 F F F 7 140-159 134 134 134 219 219 219 F F F 8 160-179 118 118 118 196 196 196 F F F 9 180-199 126 126 126 206 206 206 F F F 10 200-219 134 134 134 218 218 218 F F F 11 220-239 130 130 130 211 211 211 F F F 12 240-259 135 135 135 220 220 220 F F F 13 260-279 135 135 135 220 220 220 F F F 14 280-299 135 135 135 220 220 220 F F F 15 300-319 130 130 130 211 211 211 F F F 16 320-339 132 132 132 214 214 214 F F F 17 340-359 135 135 135 220 220 220 F F F 18 360-379 133 133 133 216 216 216 F F F 19 380-399 135 135 135 220 220 220 F F F 20 400-419 134 134 134 218 218 218 F F F 21 420-439 134 134 134 218 218 218 F F F 22 440-459 134 134 134 218 218 218 F F F 23 460-479 135 135 135 220 220 220 F F F 24 480-499 135 135 135 220 220 220 F F F 25 500-519 133 133 133 216 216 216 F F F 26 520-539 134 134 134 218 218 218 F F F 27 540-559 132 132 132 213 213 213 F F F 28 560-579 133 133 133 216 216 216 F F F 29 580-599 135 135 135 220 220 220 F F F Max gen: 135, Max flush: 220 Begin loop 68 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:51.567963787Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1677,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:51.576127212Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1677,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:51.576466112Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1677} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:51.576499941Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1677} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:51.5765089Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1677} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:51.57652066Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1677} {"msg":"Crucible b0948cb0-f737-4286-aab2-40d349f27f6e has session id: 829febd5-e436-4875-b9a4-04e6803e6bd2","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:51.576910719Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1677,"session_id":"829febd5-e436-4875-b9a4-04e6803e6bd2"} {"msg":"Upstairs opts: Upstairs UUID: b0948cb0-f737-4286-aab2-40d349f27f6e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:51.576942068Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1677,"session_id":"829febd5-e436-4875-b9a4-04e6803e6bd2"} {"msg":"Crucible stats registered with UUID: b0948cb0-f737-4286-aab2-40d349f27f6e","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:51.576953148Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1677,"session_id":"829febd5-e436-4875-b9a4-04e6803e6bd2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:46:51.576962597Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1677,"session_id":"829febd5-e436-4875-b9a4-04e6803e6bd2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:51.577117483Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1677,"":"io task","client":"2","":"downstairs","session_id":"829febd5-e436-4875-b9a4-04e6803e6bd2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:51.577136602Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1677,"":"io task","client":"0","":"downstairs","session_id":"829febd5-e436-4875-b9a4-04e6803e6bd2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:51.577146852Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1677,"":"io task","client":"1","":"downstairs","session_id":"829febd5-e436-4875-b9a4-04e6803e6bd2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:51.577275708Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1677,"task":"crutest"} {"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:51.577290398Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1677} {"msg":"b0948cb0-f737-4286-aab2-40d349f27f6e active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:51.577325607Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1677,"session_id":"829febd5-e436-4875-b9a4-04e6803e6bd2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:51.577335017Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1677,"client":"0","":"downstairs","session_id":"829febd5-e436-4875-b9a4-04e6803e6bd2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:51.577344826Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1677,"client":"1","":"downstairs","session_id":"829febd5-e436-4875-b9a4-04e6803e6bd2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:51.577354356Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1677,"client":"2","":"downstairs","session_id":"829febd5-e436-4875-b9a4-04e6803e6bd2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:51.577363866Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1677,"":"io task","client":"2","":"downstairs","session_id":"829febd5-e436-4875-b9a4-04e6803e6bd2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:51.577373896Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1677,"":"io task","client":"0","":"downstairs","session_id":"829febd5-e436-4875-b9a4-04e6803e6bd2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:51.577383845Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1677,"":"io task","client":"1","":"downstairs","session_id":"829febd5-e436-4875-b9a4-04e6803e6bd2"} {"msg":"ds_connection connected from Ok(127.0.0.1:48197)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:51.577440534Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1677,"":"io task","client":"1","":"downstairs","session_id":"829febd5-e436-4875-b9a4-04e6803e6bd2"} {"msg":"ds_connection connected from Ok(127.0.0.1:45104)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:51.577466093Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1677,"":"io task","client":"0","":"downstairs","session_id":"829febd5-e436-4875-b9a4-04e6803e6bd2"} {"msg":"ds_connection connected from Ok(127.0.0.1:63977)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:51.577502702Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1677,"":"io task","client":"2","":"downstairs","session_id":"829febd5-e436-4875-b9a4-04e6803e6bd2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:51.578154093Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1677,"client":"2","":"downstairs","session_id":"829febd5-e436-4875-b9a4-04e6803e6bd2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:51.578186592Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1677,"client":"0","":"downstairs","session_id":"829febd5-e436-4875-b9a4-04e6803e6bd2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:51.578198092Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1677,"client":"1","":"downstairs","session_id":"829febd5-e436-4875-b9a4-04e6803e6bd2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:51.578516023Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1677,"session_id":"829febd5-e436-4875-b9a4-04e6803e6bd2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:51.578558351Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1677,"session_id":"829febd5-e436-4875-b9a4-04e6803e6bd2"} {"msg":"[0]R flush_numbers[0..12]: [216, 217, 219, 216, 216, 212, 217, 219, 196, 206, 218, 211]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:51.578655089Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1677,"":"downstairs","":"downstairs","session_id":"829febd5-e436-4875-b9a4-04e6803e6bd2"} {"msg":"[0]R generation[0..12]: [133, 134, 134, 133, 133, 131, 134, 134, 118, 126, 134, 130]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:51.578666728Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1677,"":"downstairs","":"downstairs","session_id":"829febd5-e436-4875-b9a4-04e6803e6bd2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:51.578677738Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1677,"":"downstairs","":"downstairs","session_id":"829febd5-e436-4875-b9a4-04e6803e6bd2"} {"msg":"[1]R flush_numbers[0..12]: [216, 217, 219, 216, 216, 212, 217, 219, 196, 206, 218, 211]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:51.578688558Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1677,"":"downstairs","":"downstairs","session_id":"829febd5-e436-4875-b9a4-04e6803e6bd2"} {"msg":"[1]R generation[0..12]: [133, 134, 134, 133, 133, 131, 134, 134, 118, 126, 134, 130]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:51.578699087Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1677,"":"downstairs","":"downstairs","session_id":"829febd5-e436-4875-b9a4-04e6803e6bd2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:51.578709917Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1677,"":"downstairs","":"downstairs","session_id":"829febd5-e436-4875-b9a4-04e6803e6bd2"} {"msg":"[2]R flush_numbers[0..12]: [216, 217, 219, 216, 216, 212, 217, 219, 196, 206, 218, 211]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:51.578720477Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1677,"":"downstairs","":"downstairs","session_id":"829febd5-e436-4875-b9a4-04e6803e6bd2"} {"msg":"[2]R generation[0..12]: [133, 134, 134, 133, 133, 131, 134, 134, 118, 126, 134, 130]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:51.578731016Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1677,"":"downstairs","":"downstairs","session_id":"829febd5-e436-4875-b9a4-04e6803e6bd2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:51.578741956Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1677,"":"downstairs","":"downstairs","session_id":"829febd5-e436-4875-b9a4-04e6803e6bd2"} {"msg":"Max found gen is 136","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:51.578751466Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1677,"":"downstairs","":"downstairs","session_id":"829febd5-e436-4875-b9a4-04e6803e6bd2"} {"msg":"Generation requested: 136 >= found:136","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:51.578761296Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1677,"":"downstairs","":"downstairs","session_id":"829febd5-e436-4875-b9a4-04e6803e6bd2"} {"msg":"Next flush: 221","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:51.578770795Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1677,"":"downstairs","":"downstairs","session_id":"829febd5-e436-4875-b9a4-04e6803e6bd2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:51.578780305Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1677,"":"downstairs","":"downstairs","session_id":"829febd5-e436-4875-b9a4-04e6803e6bd2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:51.578790395Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1677,"session_id":"829febd5-e436-4875-b9a4-04e6803e6bd2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:51.578800684Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1677,"client":"0","":"downstairs","session_id":"829febd5-e436-4875-b9a4-04e6803e6bd2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:51.578810824Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1677,"client":"1","":"downstairs","session_id":"829febd5-e436-4875-b9a4-04e6803e6bd2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:51.578820614Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1677,"client":"2","":"downstairs","session_id":"829febd5-e436-4875-b9a4-04e6803e6bd2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:51.578830694Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1677,"session_id":"829febd5-e436-4875-b9a4-04e6803e6bd2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:51.578839913Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1677,"session_id":"829febd5-e436-4875-b9a4-04e6803e6bd2"} {"msg":"b0948cb0-f737-4286-aab2-40d349f27f6e is now active with session: 829febd5-e436-4875-b9a4-04e6803e6bd2","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:51.578849913Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1677,"session_id":"829febd5-e436-4875-b9a4-04e6803e6bd2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:51.578858663Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1677,"session_id":"829febd5-e436-4875-b9a4-04e6803e6bd2"} {"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:51.578867742Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1677} {"msg":"Activated sub_volume b0948cb0-f737-4286-aab2-40d349f27f6e","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:51.578875722Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1677} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:51.578883762Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1677,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:51.579187793Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1677,"task":"crutest"} 01/30 Write block 158 len 28672 data: 13 14 12 8 7 10 7 02/30 Write block 387 len 16384 data: 10 10 8 9 03/30 Write block 315 len 4096 data: 11 04/30 Read block 560 len 8192 05/30 Write block 545 len 8192 data: 10 9 06/30 Flush 07/30 Write block 113 len 28672 data: 11 10 11 12 11 10 11 08/30 Write block 72 len 40960 data: 12 13 13 13 12 12 14 12 9 7 09/30 Read block 57 len 24576 10/30 Read block 539 len 16384 11/30 Read block 451 len 16384 12/30 Write block 123 len 8192 data: 14 12 13/30 Write block 87 len 36864 data: 9 6 7 8 5 6 8 11 12 14/30 Read block 349 len 8192 15/30 Write block 202 len 12288 data: 16 16 17 16/30 Read block 200 len 32768 17/30 Write block 392 len 28672 data: 11 10 9 12 13 10 8 18/30 Read block 386 len 24576 19/30 Write block 169 len 20480 data: 11 11 9 10 11 20/30 Write block 549 len 4096 data: 12 21/30 Write block 488 len 20480 data: 15 15 16 16 13 22/30 Write block 306 len 20480 data: 8 8 6 9 9 23/30 Read block 196 len 20480 24/30 Read block 185 len 20480 25/30 Read block 587 len 24576 26/30 Read block 327 len 28672 27/30 Write block 201 len 24576 data: 15 17 17 18 15 14 28/30 Read block 124 len 36864 29/30 Read block 95 len 20480 30/30 Read block 388 len 36864 ---------------------------------------------------------------- Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Sent false 1001 Acked Write 4 Done Done Sent false 1002 Acked Write 1 Done Done Sent false 1003 Acked Read 2 Done Done Sent false 1004 Acked Write 2 Done Done Sent false 1005 Acked Flush 0 Done Done Sent false 1006 Acked Write 7 Done Done Sent false 1007 Acked Write 10 Done Done Sent false 1008 Acked Read 6 Done Done Sent false 1009 Acked Read 4 Done Done Sent false 1010 Acked Read 4 Done Done Sent false 1011 Acked Write 2 Done Done Sent false 1012 Acked Write 9 Done Done Sent false 1013 Acked Read 2 Done Done Sent false 1014 Acked Write 3 Done Done Sent false 1015 Acked Read 8 Done Done Sent false 1016 Acked Write 7 Done Done Sent false 1017 Acked Read 6 Done Done Sent false 1018 Acked Write 5 Done Done Sent false 1019 Acked Write 1 Done Done Sent false 1020 Acked Write 5 Done Done Sent false 1021 Acked Write 5 Done Done Sent false 1022 Acked Read 5 Done Done Sent false 1023 Acked Read 5 Done Done Sent false 1024 Acked Read 6 Done Done Sent false 1025 Acked Read 7 Done Done Sent false 1026 Acked Write 6 Done Done Sent false 1027 Acked Read 9 Done Done Sent false 1028 Acked Read 5 Done Done Sent false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 30 31 Done 30 29 0 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1005)) Some(JobId(1005)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:46:51.58726967Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1677} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:51.587453845Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1677,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 133 133 133 216 216 216 F F F 1 020-039 134 134 134 217 217 217 F F F 2 040-059 134 134 134 219 219 219 F F F 3 060-079 133 133 133 216 216 216 T T F <--- 4 080-099 133 133 133 216 216 216 T T F <--- 5 100-119 131 131 131 212 212 212 T T F <--- 6 120-139 134 134 134 217 217 217 T T F <--- 7 140-159 136 136 134 221 221 219 F F F <--- 8 160-179 136 136 118 221 221 196 T T F <--- 9 180-199 126 126 126 206 206 206 F F F 10 200-219 134 134 134 218 218 218 T T F <--- 11 220-239 130 130 130 211 211 211 F F F 12 240-259 135 135 135 220 220 220 F F F 13 260-279 135 135 135 220 220 220 F F F 14 280-299 135 135 135 220 220 220 F F F 15 300-319 136 136 130 221 221 211 T T F <--- 16 320-339 132 132 132 214 214 214 F F F 17 340-359 135 135 135 220 220 220 F F F 18 360-379 133 133 133 216 216 216 F F F 19 380-399 136 136 135 221 221 220 T T F <--- 20 400-419 134 134 134 218 218 218 F F F 21 420-439 134 134 134 218 218 218 F F F 22 440-459 134 134 134 218 218 218 F F F 23 460-479 135 135 135 220 220 220 F F F 24 480-499 135 135 135 220 220 220 T T F <--- 25 500-519 133 133 133 216 216 216 F F F 26 520-539 134 134 134 218 218 218 F F F 27 540-559 136 136 132 221 221 213 T T F <--- 28 560-579 133 133 133 216 216 216 F F F 29 580-599 135 135 135 220 220 220 F F F Max gen: 136, Max flush: 221 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 68 Verifying data now No change after verify Loop: 68 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 133 133 133 216 216 216 F F F 1 020-039 134 134 134 217 217 217 F F F 2 040-059 134 134 134 219 219 219 F F F 3 060-079 137 137 137 222 222 222 F F F 4 080-099 137 137 137 222 222 222 F F F 5 100-119 137 137 137 222 222 222 F F F 6 120-139 137 137 137 222 222 222 F F F 7 140-159 136 136 136 221 221 221 F F F 8 160-179 137 137 137 222 222 222 F F F 9 180-199 126 126 126 206 206 206 F F F 10 200-219 137 137 137 222 222 222 F F F 11 220-239 130 130 130 211 211 211 F F F 12 240-259 135 135 135 220 220 220 F F F 13 260-279 135 135 135 220 220 220 F F F 14 280-299 135 135 135 220 220 220 F F F 15 300-319 137 137 137 222 222 222 F F F 16 320-339 132 132 132 214 214 214 F F F 17 340-359 135 135 135 220 220 220 F F F 18 360-379 133 133 133 216 216 216 F F F 19 380-399 137 137 137 222 222 222 F F F 20 400-419 134 134 134 218 218 218 F F F 21 420-439 134 134 134 218 218 218 F F F 22 440-459 134 134 134 218 218 218 F F F 23 460-479 135 135 135 220 220 220 F F F 24 480-499 137 137 137 222 222 222 F F F 25 500-519 133 133 133 216 216 216 F F F 26 520-539 134 134 134 218 218 218 F F F 27 540-559 137 137 137 222 222 222 F F F 28 560-579 133 133 133 216 216 216 F F F 29 580-599 135 135 135 220 220 220 F F F Max gen: 137, Max flush: 222 Begin loop 69 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:55.831346111Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1685,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:46:55.838414677Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1685,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:55.838463296Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1685,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:55.845129593Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1685,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:55.845388096Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1685} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:55.845414905Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1685} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:55.845426605Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1685} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:55.845438164Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1685} {"msg":"Crucible 034f1618-1750-4fca-9a5a-4d332d794364 has session id: bf2d651e-4ecc-4789-b824-004e6ce3a169","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:55.845727796Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1685,"session_id":"bf2d651e-4ecc-4789-b824-004e6ce3a169"} {"msg":"Upstairs opts: Upstairs UUID: 034f1618-1750-4fca-9a5a-4d332d794364, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:55.845781564Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1685,"session_id":"bf2d651e-4ecc-4789-b824-004e6ce3a169"} {"msg":"Crucible stats registered with UUID: 034f1618-1750-4fca-9a5a-4d332d794364","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:55.845792314Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1685,"session_id":"bf2d651e-4ecc-4789-b824-004e6ce3a169"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:46:55.845801394Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1685,"session_id":"bf2d651e-4ecc-4789-b824-004e6ce3a169"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:55.84594319Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1685,"":"io task","client":"0","":"downstairs","session_id":"bf2d651e-4ecc-4789-b824-004e6ce3a169"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:55.845977399Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1685,"":"io task","client":"1","":"downstairs","session_id":"bf2d651e-4ecc-4789-b824-004e6ce3a169"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:55.845988658Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1685,"":"io task","client":"2","":"downstairs","session_id":"bf2d651e-4ecc-4789-b824-004e6ce3a169"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:55.846119185Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1685,"task":"crutest"} {"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:55.846134944Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1685} {"msg":"034f1618-1750-4fca-9a5a-4d332d794364 active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:55.846151474Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1685,"session_id":"bf2d651e-4ecc-4789-b824-004e6ce3a169"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:55.846160243Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1685,"client":"0","":"downstairs","session_id":"bf2d651e-4ecc-4789-b824-004e6ce3a169"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:55.846169843Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1685,"client":"1","":"downstairs","session_id":"bf2d651e-4ecc-4789-b824-004e6ce3a169"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:55.846181603Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1685,"client":"2","":"downstairs","session_id":"bf2d651e-4ecc-4789-b824-004e6ce3a169"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:55.846191193Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1685,"":"io task","client":"2","":"downstairs","session_id":"bf2d651e-4ecc-4789-b824-004e6ce3a169"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:55.846202552Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1685,"":"io task","client":"1","":"downstairs","session_id":"bf2d651e-4ecc-4789-b824-004e6ce3a169"} {"msg":"ds_connection connected from Ok(127.0.0.1:47901)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:55.846246811Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1685,"":"io task","client":"2","":"downstairs","session_id":"bf2d651e-4ecc-4789-b824-004e6ce3a169"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:55.846257231Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1685,"":"io task","client":"0","":"downstairs","session_id":"bf2d651e-4ecc-4789-b824-004e6ce3a169"} {"msg":"ds_connection connected from Ok(127.0.0.1:46364)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:55.846388097Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1685,"":"io task","client":"1","":"downstairs","session_id":"bf2d651e-4ecc-4789-b824-004e6ce3a169"} {"msg":"ds_connection connected from Ok(127.0.0.1:33115)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:55.846402906Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1685,"":"io task","client":"0","":"downstairs","session_id":"bf2d651e-4ecc-4789-b824-004e6ce3a169"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:55.846929201Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1685,"client":"1","":"downstairs","session_id":"bf2d651e-4ecc-4789-b824-004e6ce3a169"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:55.846955391Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1685,"client":"2","":"downstairs","session_id":"bf2d651e-4ecc-4789-b824-004e6ce3a169"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:55.847055668Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1685,"client":"0","":"downstairs","session_id":"bf2d651e-4ecc-4789-b824-004e6ce3a169"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:55.847251542Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1685,"session_id":"bf2d651e-4ecc-4789-b824-004e6ce3a169"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:55.847261112Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1685,"session_id":"bf2d651e-4ecc-4789-b824-004e6ce3a169"} {"msg":"[0]R flush_numbers[0..12]: [216, 217, 219, 222, 222, 222, 222, 221, 222, 206, 222, 211]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:55.847339639Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1685,"":"downstairs","":"downstairs","session_id":"bf2d651e-4ecc-4789-b824-004e6ce3a169"} {"msg":"[0]R generation[0..12]: [133, 134, 134, 137, 137, 137, 137, 136, 137, 126, 137, 130]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:55.847350729Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1685,"":"downstairs","":"downstairs","session_id":"bf2d651e-4ecc-4789-b824-004e6ce3a169"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:55.847361639Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1685,"":"downstairs","":"downstairs","session_id":"bf2d651e-4ecc-4789-b824-004e6ce3a169"} {"msg":"[1]R flush_numbers[0..12]: [216, 217, 219, 222, 222, 222, 222, 221, 222, 206, 222, 211]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:55.847374308Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1685,"":"downstairs","":"downstairs","session_id":"bf2d651e-4ecc-4789-b824-004e6ce3a169"} {"msg":"[1]R generation[0..12]: [133, 134, 134, 137, 137, 137, 137, 136, 137, 126, 137, 130]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:55.847384838Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1685,"":"downstairs","":"downstairs","session_id":"bf2d651e-4ecc-4789-b824-004e6ce3a169"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:55.847395668Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1685,"":"downstairs","":"downstairs","session_id":"bf2d651e-4ecc-4789-b824-004e6ce3a169"} {"msg":"[2]R flush_numbers[0..12]: [216, 217, 219, 222, 222, 222, 222, 221, 222, 206, 222, 211]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:55.847406118Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1685,"":"downstairs","":"downstairs","session_id":"bf2d651e-4ecc-4789-b824-004e6ce3a169"} {"msg":"[2]R generation[0..12]: [133, 134, 134, 137, 137, 137, 137, 136, 137, 126, 137, 130]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:55.847416647Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1685,"":"downstairs","":"downstairs","session_id":"bf2d651e-4ecc-4789-b824-004e6ce3a169"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:55.847427397Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1685,"":"downstairs","":"downstairs","session_id":"bf2d651e-4ecc-4789-b824-004e6ce3a169"} {"msg":"Max found gen is 138","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:55.847436717Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1685,"":"downstairs","":"downstairs","session_id":"bf2d651e-4ecc-4789-b824-004e6ce3a169"} {"msg":"Generation requested: 138 >= found:138","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:55.847446416Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1685,"":"downstairs","":"downstairs","session_id":"bf2d651e-4ecc-4789-b824-004e6ce3a169"} {"msg":"Next flush: 223","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:55.847455846Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1685,"":"downstairs","":"downstairs","session_id":"bf2d651e-4ecc-4789-b824-004e6ce3a169"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:55.847465296Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1685,"":"downstairs","":"downstairs","session_id":"bf2d651e-4ecc-4789-b824-004e6ce3a169"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:55.847474916Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1685,"session_id":"bf2d651e-4ecc-4789-b824-004e6ce3a169"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:55.847483685Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1685,"client":"0","":"downstairs","session_id":"bf2d651e-4ecc-4789-b824-004e6ce3a169"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:55.847493375Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1685,"client":"1","":"downstairs","session_id":"bf2d651e-4ecc-4789-b824-004e6ce3a169"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:55.847502925Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1685,"client":"2","":"downstairs","session_id":"bf2d651e-4ecc-4789-b824-004e6ce3a169"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:55.847512604Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1685,"session_id":"bf2d651e-4ecc-4789-b824-004e6ce3a169"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:55.847523004Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1685,"session_id":"bf2d651e-4ecc-4789-b824-004e6ce3a169"} {"msg":"034f1618-1750-4fca-9a5a-4d332d794364 is now active with session: bf2d651e-4ecc-4789-b824-004e6ce3a169","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:55.847532844Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1685,"session_id":"bf2d651e-4ecc-4789-b824-004e6ce3a169"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:55.847541524Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1685,"session_id":"bf2d651e-4ecc-4789-b824-004e6ce3a169"} {"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:55.847550583Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1685} {"msg":"Activated sub_volume 034f1618-1750-4fca-9a5a-4d332d794364","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:55.847558473Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1685} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:55.847566543Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1685,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:55.847868824Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1685,"task":"crutest"} 01/30 Write block 499 len 28672 data: 9 12 13 13 14 14 13 02/30 Read block 562 len 8192 03/30 Read block 248 len 32768 04/30 Read block 506 len 8192 05/30 Read block 365 len 8192 06/30 Read block 247 len 20480 07/30 Write block 112 len 4096 data: 11 08/30 Write block 272 len 20480 data: 8 10 7 8 10 09/30 Read block 570 len 16384 10/30 Read block 97 len 28672 11/30 Write block 373 len 12288 data: 7 6 10 12/30 Read block 87 len 16384 13/30 Read block 116 len 28672 14/30 Read block 375 len 12288 15/30 Read block 330 len 8192 16/30 Read block 62 len 40960 17/30 Write block 544 len 8192 data: 12 11 18/30 Read block 417 len 12288 19/30 Read block 502 len 36864 20/30 Read block 450 len 36864 21/30 Flush 22/30 Write block 410 len 36864 data: 11 14 14 14 11 8 13 13 12 23/30 Read block 211 len 36864 24/30 Write block 125 len 4096 data: 11 25/30 Read block 445 len 12288 26/30 Read block 99 len 20480 27/30 Read block 72 len 16384 28/30 Write block 497 len 36864 data: 7 9 10 13 14 14 15 15 14 29/30 Read block 287 len 24576 30/30 Read block 264 len 12288 ---------------------------------------------------------------- Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Read 2 Done Done Done false 1002 Acked Read 8 Done Done Done false 1003 Acked Read 2 Done Done Done false 1004 Acked Read 2 Done Done Done false 1005 Acked Read 5 Done Done Done false 1006 Acked Write 1 Done Done Done false 1007 Acked Write 5 Done Done Done false 1008 Acked Read 4 Done Done Done false 1009 Acked Read 7 Done Done Done false 1010 Acked Write 3 Done Done Done false 1011 Acked Read 4 Done Done Done false 1012 Acked Read 7 Done Done Done false 1013 Acked Read 3 Done Done Done false 1014 Acked Read 2 Sent Done Done false 1015 Acked Read 10 Sent Done Done false 1016 Acked Write 2 Sent Done Done false 1017 Acked Read 3 Sent Done Done false 1018 Acked Read 9 Sent Done Done false 1019 Acked Read 9 Sent Done Done false 1020 Acked Flush 0 Sent Done Done false 1021 Acked Write 9 Sent Done Done false 1022 Acked Read 9 Sent Done Done false 1023 Acked Write 1 Sent Done Done false 1024 Acked Read 3 Sent Done Done false 1025 Acked Read 5 Sent Done Done false 1026 Acked Read 4 Sent Done Done false 1027 Acked Write 9 Sent Done Done false 1028 Acked Read 6 Sent Done Done false 1029 Acked Read 3 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 16 0 0 16 Done 14 30 30 74 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:46:55.857234774Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1685} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:46:55.857416349Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1685,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 133 133 133 216 216 216 F F F 1 020-039 134 134 134 217 217 217 F F F 2 040-059 134 134 134 219 219 219 F F F 3 060-079 137 137 137 222 222 222 F F F 4 080-099 137 137 137 222 222 222 F F F 5 100-119 137 138 138 222 223 223 T F F <--- 6 120-139 137 137 137 222 222 222 F T T <--- 7 140-159 136 136 136 221 221 221 F F F 8 160-179 137 137 137 222 222 222 F F F 9 180-199 126 126 126 206 206 206 F F F 10 200-219 137 137 137 222 222 222 F F F 11 220-239 130 130 130 211 211 211 F F F 12 240-259 135 135 135 220 220 220 F F F 13 260-279 135 138 138 220 223 223 T F F <--- 14 280-299 135 135 135 220 220 220 F F F 15 300-319 137 137 137 222 222 222 F F F 16 320-339 132 132 132 214 214 214 F F F 17 340-359 135 135 135 220 220 220 F F F 18 360-379 133 138 138 216 223 223 T F F <--- 19 380-399 137 137 137 222 222 222 F F F 20 400-419 134 134 134 218 218 218 F T T <--- 21 420-439 134 134 134 218 218 218 F F F 22 440-459 134 134 134 218 218 218 F F F 23 460-479 135 135 135 220 220 220 F F F 24 480-499 137 138 138 222 223 223 T T T <--- 25 500-519 133 138 138 216 223 223 T T T <--- 26 520-539 134 134 134 218 218 218 F F F 27 540-559 137 138 138 222 223 223 F F F <--- 28 560-579 133 133 133 216 216 216 F F F 29 580-599 135 135 135 220 220 220 F F F Max gen: 138, Max flush: 223 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 69 Verifying data now No change after verify Loop: 69 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 133 133 133 216 216 216 F F F 1 020-039 134 134 134 217 217 217 F F F 2 040-059 134 134 134 219 219 219 F F F 3 060-079 137 137 137 222 222 222 F F F 4 080-099 137 137 137 222 222 222 F F F 5 100-119 138 138 138 223 223 223 F F F 6 120-139 139 139 139 224 224 224 F F F 7 140-159 136 136 136 221 221 221 F F F 8 160-179 137 137 137 222 222 222 F F F 9 180-199 126 126 126 206 206 206 F F F 10 200-219 137 137 137 222 222 222 F F F 11 220-239 130 130 130 211 211 211 F F F 12 240-259 135 135 135 220 220 220 F F F 13 260-279 138 138 138 223 223 223 F F F 14 280-299 135 135 135 220 220 220 F F F 15 300-319 137 137 137 222 222 222 F F F 16 320-339 132 132 132 214 214 214 F F F 17 340-359 135 135 135 220 220 220 F F F 18 360-379 138 138 138 223 223 223 F F F 19 380-399 137 137 137 222 222 222 F F F 20 400-419 139 139 139 224 224 224 F F F 21 420-439 134 134 134 218 218 218 F F F 22 440-459 134 134 134 218 218 218 F F F 23 460-479 135 135 135 220 220 220 F F F 24 480-499 139 139 139 224 224 224 F F F 25 500-519 139 139 139 224 224 224 F F F 26 520-539 134 134 134 218 218 218 F F F 27 540-559 138 138 138 223 223 223 F F F 28 560-579 133 133 133 216 216 216 F F F 29 580-599 135 135 135 220 220 220 F F F Max gen: 139, Max flush: 224 Begin loop 70 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:00.106814117Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1693,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:00.114866844Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1693,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:00.115145646Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1693} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:00.115174705Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1693} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:00.115183425Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1693} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:00.115194915Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1693} {"msg":"Crucible 513cf18c-5d3c-4d95-a30b-4b3345d5b4e4 has session id: 4053bca8-a11b-4efc-95a5-05e5d9de49c2","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:00.115510896Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1693,"session_id":"4053bca8-a11b-4efc-95a5-05e5d9de49c2"} {"msg":"Upstairs opts: Upstairs UUID: 513cf18c-5d3c-4d95-a30b-4b3345d5b4e4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:00.115539205Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1693,"session_id":"4053bca8-a11b-4efc-95a5-05e5d9de49c2"} {"msg":"Crucible stats registered with UUID: 513cf18c-5d3c-4d95-a30b-4b3345d5b4e4","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:00.115551904Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1693,"session_id":"4053bca8-a11b-4efc-95a5-05e5d9de49c2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:47:00.115560874Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1693,"session_id":"4053bca8-a11b-4efc-95a5-05e5d9de49c2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:00.11570962Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1693,"":"io task","client":"2","":"downstairs","session_id":"4053bca8-a11b-4efc-95a5-05e5d9de49c2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:00.115727819Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1693,"":"io task","client":"1","":"downstairs","session_id":"4053bca8-a11b-4efc-95a5-05e5d9de49c2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:00.115737979Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1693,"":"io task","client":"0","":"downstairs","session_id":"4053bca8-a11b-4efc-95a5-05e5d9de49c2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:00.115872855Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1693,"task":"crutest"} {"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:00.115884545Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1693} {"msg":"513cf18c-5d3c-4d95-a30b-4b3345d5b4e4 active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:00.115901034Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1693,"session_id":"4053bca8-a11b-4efc-95a5-05e5d9de49c2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:00.115909874Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1693,"client":"0","":"downstairs","session_id":"4053bca8-a11b-4efc-95a5-05e5d9de49c2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:00.115919644Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1693,"client":"1","":"downstairs","session_id":"4053bca8-a11b-4efc-95a5-05e5d9de49c2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:00.115929174Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1693,"client":"2","":"downstairs","session_id":"4053bca8-a11b-4efc-95a5-05e5d9de49c2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:00.115938813Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1693,"":"io task","client":"2","":"downstairs","session_id":"4053bca8-a11b-4efc-95a5-05e5d9de49c2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:00.115948583Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1693,"":"io task","client":"0","":"downstairs","session_id":"4053bca8-a11b-4efc-95a5-05e5d9de49c2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:00.115958563Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1693,"":"io task","client":"1","":"downstairs","session_id":"4053bca8-a11b-4efc-95a5-05e5d9de49c2"} {"msg":"ds_connection connected from Ok(127.0.0.1:48811)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:00.11604061Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1693,"":"io task","client":"2","":"downstairs","session_id":"4053bca8-a11b-4efc-95a5-05e5d9de49c2"} {"msg":"ds_connection connected from Ok(127.0.0.1:56064)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:00.116107838Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1693,"":"io task","client":"0","":"downstairs","session_id":"4053bca8-a11b-4efc-95a5-05e5d9de49c2"} {"msg":"ds_connection connected from Ok(127.0.0.1:58386)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:00.116122908Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1693,"":"io task","client":"1","":"downstairs","session_id":"4053bca8-a11b-4efc-95a5-05e5d9de49c2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:00.11674129Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1693,"client":"0","":"downstairs","session_id":"4053bca8-a11b-4efc-95a5-05e5d9de49c2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:00.11676057Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1693,"client":"2","":"downstairs","session_id":"4053bca8-a11b-4efc-95a5-05e5d9de49c2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:00.116771399Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1693,"client":"1","":"downstairs","session_id":"4053bca8-a11b-4efc-95a5-05e5d9de49c2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:00.11708026Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1693,"session_id":"4053bca8-a11b-4efc-95a5-05e5d9de49c2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:00.11709249Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1693,"session_id":"4053bca8-a11b-4efc-95a5-05e5d9de49c2"} {"msg":"[0]R flush_numbers[0..12]: [216, 217, 219, 222, 222, 223, 224, 221, 222, 206, 222, 211]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:00.117134569Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1693,"":"downstairs","":"downstairs","session_id":"4053bca8-a11b-4efc-95a5-05e5d9de49c2"} {"msg":"[0]R generation[0..12]: [133, 134, 134, 137, 137, 138, 139, 136, 137, 126, 137, 130]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:00.117145769Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1693,"":"downstairs","":"downstairs","session_id":"4053bca8-a11b-4efc-95a5-05e5d9de49c2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:00.117156678Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1693,"":"downstairs","":"downstairs","session_id":"4053bca8-a11b-4efc-95a5-05e5d9de49c2"} {"msg":"[1]R flush_numbers[0..12]: [216, 217, 219, 222, 222, 223, 224, 221, 222, 206, 222, 211]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:00.117167298Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1693,"":"downstairs","":"downstairs","session_id":"4053bca8-a11b-4efc-95a5-05e5d9de49c2"} {"msg":"[1]R generation[0..12]: [133, 134, 134, 137, 137, 138, 139, 136, 137, 126, 137, 130]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:00.117177768Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1693,"":"downstairs","":"downstairs","session_id":"4053bca8-a11b-4efc-95a5-05e5d9de49c2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:00.117188587Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1693,"":"downstairs","":"downstairs","session_id":"4053bca8-a11b-4efc-95a5-05e5d9de49c2"} {"msg":"[2]R flush_numbers[0..12]: [216, 217, 219, 222, 222, 223, 224, 221, 222, 206, 222, 211]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:00.117199137Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1693,"":"downstairs","":"downstairs","session_id":"4053bca8-a11b-4efc-95a5-05e5d9de49c2"} {"msg":"[2]R generation[0..12]: [133, 134, 134, 137, 137, 138, 139, 136, 137, 126, 137, 130]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:00.117209677Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1693,"":"downstairs","":"downstairs","session_id":"4053bca8-a11b-4efc-95a5-05e5d9de49c2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:00.117222256Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1693,"":"downstairs","":"downstairs","session_id":"4053bca8-a11b-4efc-95a5-05e5d9de49c2"} {"msg":"Max found gen is 140","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:00.117231586Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1693,"":"downstairs","":"downstairs","session_id":"4053bca8-a11b-4efc-95a5-05e5d9de49c2"} {"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:00.117241356Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1693,"":"downstairs","":"downstairs","session_id":"4053bca8-a11b-4efc-95a5-05e5d9de49c2"} {"msg":"Next flush: 225","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:00.117250655Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1693,"":"downstairs","":"downstairs","session_id":"4053bca8-a11b-4efc-95a5-05e5d9de49c2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:00.117259905Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1693,"":"downstairs","":"downstairs","session_id":"4053bca8-a11b-4efc-95a5-05e5d9de49c2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:00.117269605Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1693,"session_id":"4053bca8-a11b-4efc-95a5-05e5d9de49c2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:00.117278275Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1693,"client":"0","":"downstairs","session_id":"4053bca8-a11b-4efc-95a5-05e5d9de49c2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:00.117288054Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1693,"client":"1","":"downstairs","session_id":"4053bca8-a11b-4efc-95a5-05e5d9de49c2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:00.117297704Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1693,"client":"2","":"downstairs","session_id":"4053bca8-a11b-4efc-95a5-05e5d9de49c2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:00.117307414Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1693,"session_id":"4053bca8-a11b-4efc-95a5-05e5d9de49c2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:00.117316404Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1693,"session_id":"4053bca8-a11b-4efc-95a5-05e5d9de49c2"} {"msg":"513cf18c-5d3c-4d95-a30b-4b3345d5b4e4 is now active with session: 4053bca8-a11b-4efc-95a5-05e5d9de49c2","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:00.117326353Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1693,"session_id":"4053bca8-a11b-4efc-95a5-05e5d9de49c2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:00.117335123Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1693,"session_id":"4053bca8-a11b-4efc-95a5-05e5d9de49c2"} {"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:00.117344223Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1693} {"msg":"Activated sub_volume 513cf18c-5d3c-4d95-a30b-4b3345d5b4e4","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:00.117352123Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1693} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:00.117360132Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1693,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:00.117707052Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1693,"task":"crutest"} 01/30 Write block 354 len 40960 data: 11 13 11 11 10 9 10 12 15 16 02/30 Read block 452 len 16384 03/30 Read block 566 len 16384 04/30 Read block 401 len 20480 05/30 Flush 06/30 Write block 504 len 16384 data: 16 15 12 13 07/30 Read block 26 len 8192 08/30 Read block 1 len 28672 09/30 Flush 10/30 Write block 552 len 36864 data: 11 12 9 8 7 10 13 12 12 11/30 Flush 12/30 Write block 579 len 28672 data: 10 10 10 13 13 12 12 13/30 Read block 471 len 12288 14/30 Flush 15/30 Write block 507 len 32768 data: 14 8 8 10 16 11 7 7 16/30 Write block 449 len 12288 data: 8 10 8 17/30 Read block 470 len 20480 18/30 Write block 476 len 40960 data: 15 14 14 13 11 14 10 9 9 12 19/30 Write block 509 len 24576 data: 9 11 17 12 8 8 20/30 Write block 7 len 20480 data: 12 13 11 11 10 21/30 Read block 470 len 32768 22/30 Read block 383 len 12288 23/30 Read block 444 len 4096 24/30 Write block 462 len 40960 data: 6 7 6 4 5 5 6 7 8 6 25/30 Read block 20 len 20480 26/30 Read block 229 len 24576 27/30 Write block 305 len 28672 data: 10 9 9 7 10 10 10 28/30 Read block 379 len 28672 29/30 Write block 77 len 12288 data: 13 15 13 30/30 Read block 415 len 12288 ---------------------------------------------------------------- Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Sent Done false 1001 Acked Read 4 Done Sent Done false 1002 Acked Read 4 Done Sent Done false 1003 Acked Read 5 Done Sent Done false 1004 Acked Flush 0 Done Sent Done false 1005 Acked Write 4 Done Sent Done false 1006 Acked Read 2 Done Sent Done false 1007 Acked Read 7 Done Sent Done false 1008 Acked Flush 0 Done Sent Done false 1009 Acked Write 9 Done Sent Done false 1010 Acked Flush 0 Done Sent Done false 1011 Acked Write 7 Done Sent Done false 1012 Acked Read 3 Done Sent Done false 1013 Acked Flush 0 Done Sent Done false 1014 Acked Write 8 Done Sent Done false 1015 Acked Write 3 Done Sent Done false 1016 Acked Read 5 Done Sent Done false 1017 Acked Write 10 Done Sent Done false 1018 Acked Write 6 Done Sent Done false 1019 Acked Write 5 Done Sent Done false 1020 Acked Read 8 Done Sent Done false 1021 Acked Read 3 Done Sent Done false 1022 Acked Read 1 Done Sent Done false 1023 Acked Write 10 Done Sent Done false 1024 Acked Read 5 Done Sent Done false 1025 Acked Read 6 Done Sent Done false 1026 Acked Write 7 Done Sent Done false 1027 Acked Read 7 Done Sent Done false 1028 Acked Write 3 Done Sent Done false 1029 Acked Read 3 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 30 0 31 Done 29 0 30 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1013)) None Some(JobId(1013)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:47:00.127367644Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1693} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:00.127554278Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1693,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 133 133 133 216 216 216 T F T <--- 1 020-039 134 134 134 217 217 217 F F F 2 040-059 134 134 134 219 219 219 F F F 3 060-079 137 137 137 222 222 222 T F T <--- 4 080-099 137 137 137 222 222 222 F F F 5 100-119 138 138 138 223 223 223 F F F 6 120-139 139 139 139 224 224 224 F F F 7 140-159 136 136 136 221 221 221 F F F 8 160-179 137 137 137 222 222 222 F F F 9 180-199 126 126 126 206 206 206 F F F 10 200-219 137 137 137 222 222 222 F F F 11 220-239 130 130 130 211 211 211 F F F 12 240-259 135 135 135 220 220 220 F F F 13 260-279 138 138 138 223 223 223 F F F 14 280-299 135 135 135 220 220 220 F F F 15 300-319 137 137 137 222 222 222 T F T <--- 16 320-339 132 132 132 214 214 214 F F F 17 340-359 140 135 140 225 220 225 F F F <--- 18 360-379 140 138 140 225 223 225 F F F <--- 19 380-399 137 137 137 222 222 222 F F F 20 400-419 139 139 139 224 224 224 F F F 21 420-439 134 134 134 218 218 218 F F F 22 440-459 134 134 134 218 218 218 T F T <--- 23 460-479 135 135 135 220 220 220 T F T <--- 24 480-499 139 139 139 224 224 224 T F T <--- 25 500-519 140 139 140 226 224 226 T F T <--- 26 520-539 134 134 134 218 218 218 F F F 27 540-559 140 138 140 227 223 227 F F F <--- 28 560-579 140 133 140 228 216 228 F F F <--- 29 580-599 140 135 140 228 220 228 F F F <--- Max gen: 140, Max flush: 228 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 70 Verifying data now No change after verify Loop: 70 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 141 141 141 229 229 229 F F F 1 020-039 134 134 134 217 217 217 F F F 2 040-059 134 134 134 219 219 219 F F F 3 060-079 141 141 141 229 229 229 F F F 4 080-099 137 137 137 222 222 222 F F F 5 100-119 138 138 138 223 223 223 F F F 6 120-139 139 139 139 224 224 224 F F F 7 140-159 136 136 136 221 221 221 F F F 8 160-179 137 137 137 222 222 222 F F F 9 180-199 126 126 126 206 206 206 F F F 10 200-219 137 137 137 222 222 222 F F F 11 220-239 130 130 130 211 211 211 F F F 12 240-259 135 135 135 220 220 220 F F F 13 260-279 138 138 138 223 223 223 F F F 14 280-299 135 135 135 220 220 220 F F F 15 300-319 141 141 141 229 229 229 F F F 16 320-339 132 132 132 214 214 214 F F F 17 340-359 140 140 140 225 225 225 F F F 18 360-379 140 140 140 225 225 225 F F F 19 380-399 137 137 137 222 222 222 F F F 20 400-419 139 139 139 224 224 224 F F F 21 420-439 134 134 134 218 218 218 F F F 22 440-459 141 141 141 229 229 229 F F F 23 460-479 141 141 141 229 229 229 F F F 24 480-499 141 141 141 229 229 229 F F F 25 500-519 141 141 141 229 229 229 F F F 26 520-539 134 134 134 218 218 218 F F F 27 540-559 140 140 140 227 227 227 F F F 28 560-579 140 140 140 228 228 228 F F F 29 580-599 140 140 140 228 228 228 F F F Max gen: 141, Max flush: 229 Begin loop 71 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:04.37196844Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1701,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:04.37991786Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1701,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:04.380208132Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1701} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:04.380235411Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1701} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:04.380244111Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1701} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:04.380255551Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1701} {"msg":"Crucible d154460b-dd0c-4a1c-976c-958000e0542b has session id: a91412be-436a-473a-a0d2-56e820b810c9","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:04.380601501Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1701,"session_id":"a91412be-436a-473a-a0d2-56e820b810c9"} {"msg":"Upstairs opts: Upstairs UUID: d154460b-dd0c-4a1c-976c-958000e0542b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:04.38063063Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1701,"session_id":"a91412be-436a-473a-a0d2-56e820b810c9"} {"msg":"Crucible stats registered with UUID: d154460b-dd0c-4a1c-976c-958000e0542b","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:04.38064182Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1701,"session_id":"a91412be-436a-473a-a0d2-56e820b810c9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:47:04.380651009Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1701,"session_id":"a91412be-436a-473a-a0d2-56e820b810c9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:04.380765196Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1701,"":"io task","client":"2","":"downstairs","session_id":"a91412be-436a-473a-a0d2-56e820b810c9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:04.380782486Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1701,"":"io task","client":"0","":"downstairs","session_id":"a91412be-436a-473a-a0d2-56e820b810c9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:04.380792475Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1701,"":"io task","client":"1","":"downstairs","session_id":"a91412be-436a-473a-a0d2-56e820b810c9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:04.380934131Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1701,"task":"crutest"} {"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:04.380944661Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1701} {"msg":"d154460b-dd0c-4a1c-976c-958000e0542b active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:04.38096807Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1701,"session_id":"a91412be-436a-473a-a0d2-56e820b810c9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:04.38097744Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1701,"client":"0","":"downstairs","session_id":"a91412be-436a-473a-a0d2-56e820b810c9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:04.38098726Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1701,"client":"1","":"downstairs","session_id":"a91412be-436a-473a-a0d2-56e820b810c9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:04.380996819Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1701,"client":"2","":"downstairs","session_id":"a91412be-436a-473a-a0d2-56e820b810c9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:04.381006489Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1701,"":"io task","client":"0","":"downstairs","session_id":"a91412be-436a-473a-a0d2-56e820b810c9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:04.381016689Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1701,"":"io task","client":"2","":"downstairs","session_id":"a91412be-436a-473a-a0d2-56e820b810c9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:04.381026568Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1701,"":"io task","client":"1","":"downstairs","session_id":"a91412be-436a-473a-a0d2-56e820b810c9"} {"msg":"ds_connection connected from Ok(127.0.0.1:48883)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:04.381078827Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1701,"":"io task","client":"2","":"downstairs","session_id":"a91412be-436a-473a-a0d2-56e820b810c9"} {"msg":"ds_connection connected from Ok(127.0.0.1:37729)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:04.381103976Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1701,"":"io task","client":"1","":"downstairs","session_id":"a91412be-436a-473a-a0d2-56e820b810c9"} {"msg":"ds_connection connected from Ok(127.0.0.1:59463)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:04.381115326Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1701,"":"io task","client":"0","":"downstairs","session_id":"a91412be-436a-473a-a0d2-56e820b810c9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:04.381753727Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1701,"client":"0","":"downstairs","session_id":"a91412be-436a-473a-a0d2-56e820b810c9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:04.381776497Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1701,"client":"1","":"downstairs","session_id":"a91412be-436a-473a-a0d2-56e820b810c9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:04.381829755Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1701,"client":"2","":"downstairs","session_id":"a91412be-436a-473a-a0d2-56e820b810c9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:04.382057239Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1701,"session_id":"a91412be-436a-473a-a0d2-56e820b810c9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:04.382084158Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1701,"session_id":"a91412be-436a-473a-a0d2-56e820b810c9"} {"msg":"[0]R flush_numbers[0..12]: [229, 217, 219, 229, 222, 223, 224, 221, 222, 206, 222, 211]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:04.382120387Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1701,"":"downstairs","":"downstairs","session_id":"a91412be-436a-473a-a0d2-56e820b810c9"} {"msg":"[0]R generation[0..12]: [141, 134, 134, 141, 137, 138, 139, 136, 137, 126, 137, 130]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:04.382131537Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1701,"":"downstairs","":"downstairs","session_id":"a91412be-436a-473a-a0d2-56e820b810c9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:04.382142436Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1701,"":"downstairs","":"downstairs","session_id":"a91412be-436a-473a-a0d2-56e820b810c9"} {"msg":"[1]R flush_numbers[0..12]: [229, 217, 219, 229, 222, 223, 224, 221, 222, 206, 222, 211]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:04.382158096Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1701,"":"downstairs","":"downstairs","session_id":"a91412be-436a-473a-a0d2-56e820b810c9"} {"msg":"[1]R generation[0..12]: [141, 134, 134, 141, 137, 138, 139, 136, 137, 126, 137, 130]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:04.382168786Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1701,"":"downstairs","":"downstairs","session_id":"a91412be-436a-473a-a0d2-56e820b810c9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:04.382179585Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1701,"":"downstairs","":"downstairs","session_id":"a91412be-436a-473a-a0d2-56e820b810c9"} {"msg":"[2]R flush_numbers[0..12]: [229, 217, 219, 229, 222, 223, 224, 221, 222, 206, 222, 211]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:04.382190385Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1701,"":"downstairs","":"downstairs","session_id":"a91412be-436a-473a-a0d2-56e820b810c9"} {"msg":"[2]R generation[0..12]: [141, 134, 134, 141, 137, 138, 139, 136, 137, 126, 137, 130]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:04.382200865Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1701,"":"downstairs","":"downstairs","session_id":"a91412be-436a-473a-a0d2-56e820b810c9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:04.382211704Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1701,"":"downstairs","":"downstairs","session_id":"a91412be-436a-473a-a0d2-56e820b810c9"} {"msg":"Max found gen is 142","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:04.382221034Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1701,"":"downstairs","":"downstairs","session_id":"a91412be-436a-473a-a0d2-56e820b810c9"} {"msg":"Generation requested: 142 >= found:142","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:04.382230754Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1701,"":"downstairs","":"downstairs","session_id":"a91412be-436a-473a-a0d2-56e820b810c9"} {"msg":"Next flush: 230","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:04.382240063Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1701,"":"downstairs","":"downstairs","session_id":"a91412be-436a-473a-a0d2-56e820b810c9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:04.382249463Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1701,"":"downstairs","":"downstairs","session_id":"a91412be-436a-473a-a0d2-56e820b810c9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:04.382260853Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1701,"session_id":"a91412be-436a-473a-a0d2-56e820b810c9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:04.382269623Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1701,"client":"0","":"downstairs","session_id":"a91412be-436a-473a-a0d2-56e820b810c9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:04.382279082Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1701,"client":"1","":"downstairs","session_id":"a91412be-436a-473a-a0d2-56e820b810c9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:04.382288592Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1701,"client":"2","":"downstairs","session_id":"a91412be-436a-473a-a0d2-56e820b810c9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:04.382298312Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1701,"session_id":"a91412be-436a-473a-a0d2-56e820b810c9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:04.382307282Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1701,"session_id":"a91412be-436a-473a-a0d2-56e820b810c9"} {"msg":"d154460b-dd0c-4a1c-976c-958000e0542b is now active with session: a91412be-436a-473a-a0d2-56e820b810c9","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:04.382317051Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1701,"session_id":"a91412be-436a-473a-a0d2-56e820b810c9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:04.382325671Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1701,"session_id":"a91412be-436a-473a-a0d2-56e820b810c9"} {"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:04.382334731Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1701} {"msg":"Activated sub_volume d154460b-dd0c-4a1c-976c-958000e0542b","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:04.38234285Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1701} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:04.38235114Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1701,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:04.382661381Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1701,"task":"crutest"} 01/30 Write block 258 len 8192 data: 11 11 02/30 Read block 210 len 16384 03/30 Read block 413 len 32768 04/30 Write block 310 len 16384 data: 11 11 11 11 05/30 Read block 229 len 24576 06/30 Read block 308 len 20480 07/30 Read block 386 len 20480 08/30 Write block 324 len 16384 data: 16 15 16 17 09/30 Read block 542 len 28672 10/30 Read block 305 len 36864 11/30 Read block 289 len 4096 12/30 Read block 509 len 32768 13/30 Write block 180 len 12288 data: 10 9 10 14/30 Flush 15/30 Write block 372 len 8192 data: 7 8 16/30 Flush 17/30 Write block 474 len 32768 data: 8 11 16 15 15 14 12 15 18/30 Read block 508 len 12288 19/30 Read block 514 len 40960 20/30 Write block 324 len 40960 data: 17 16 17 18 14 13 11 12 10 10 21/30 Write block 128 len 32768 data: 13 11 13 13 12 12 12 11 22/30 Write block 313 len 12288 data: 12 12 12 23/30 Write block 535 len 24576 data: 17 15 18 20 19 18 24/30 Write block 493 len 4096 data: 11 25/30 Read block 177 len 32768 26/30 Flush 27/30 Write block 411 len 4096 data: 15 28/30 Read block 292 len 4096 29/30 Write block 145 len 16384 data: 13 14 13 11 30/30 Write block 487 len 8192 data: 14 16 ---------------------------------------------------------------- Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Read 4 Done Sent Done false 1002 Acked Read 8 Done Sent Done false 1003 Acked Write 4 Done Sent Done false 1004 Acked Read 6 Done Sent Done false 1005 Acked Read 5 Done Sent Done false 1006 Acked Read 5 Done Sent Done false 1007 Acked Write 4 Done Sent Done false 1008 Acked Read 7 Done Sent Done false 1009 Acked Read 9 Done Sent Done false 1010 Acked Read 1 Done Sent Done false 1011 Acked Read 8 Done Sent Done false 1012 Acked Write 3 Done Sent Done false 1013 Acked Flush 0 Done Sent Done false 1014 Acked Write 2 Done Sent Done false 1015 Acked Flush 0 Done Sent Done false 1016 Acked Write 8 Done Sent Done false 1017 Acked Read 3 Done Sent Done false 1018 Acked Read 10 Done Sent Done false 1019 Acked Write 10 Done Sent Done false 1020 Acked Write 8 Done Sent Done false 1021 Acked Write 3 Done Sent Done false 1022 Acked Write 6 Done Sent Done false 1023 Acked Write 1 Done Sent Done false 1024 Acked Read 8 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 1 Done Sent Done false 1027 Acked Read 1 Done Sent Done false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 29 2 33 Done 28 1 28 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) None Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:47:04.390780737Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1701} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:04.390958982Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1701,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 141 141 141 229 229 229 F F F 1 020-039 134 134 134 217 217 217 F F F 2 040-059 134 134 134 219 219 219 F F F 3 060-079 141 141 141 229 229 229 F F F 4 080-099 137 137 137 222 222 222 F F F 5 100-119 138 138 138 223 223 223 F F F 6 120-139 142 139 142 232 224 232 F F F <--- 7 140-159 136 136 136 221 221 221 T F T <--- 8 160-179 137 137 137 222 222 222 F F F 9 180-199 142 126 142 230 206 230 F F F <--- 10 200-219 137 137 137 222 222 222 F F F 11 220-239 130 130 130 211 211 211 F F F 12 240-259 142 135 142 230 220 230 F T F <--- 13 260-279 138 138 138 223 223 223 F F F 14 280-299 135 135 135 220 220 220 F F F 15 300-319 142 141 142 232 229 232 F F F <--- 16 320-339 142 132 142 232 214 232 F F F <--- 17 340-359 140 140 140 225 225 225 F F F 18 360-379 142 140 142 231 225 231 F F F <--- 19 380-399 137 137 137 222 222 222 F F F 20 400-419 139 139 139 224 224 224 T F T <--- 21 420-439 134 134 134 218 218 218 F F F 22 440-459 141 141 141 229 229 229 F F F 23 460-479 142 141 142 232 229 232 F F F <--- 24 480-499 142 141 142 232 229 232 T F T <--- 25 500-519 141 141 141 229 229 229 F F F 26 520-539 142 134 142 232 218 232 F F F <--- 27 540-559 142 140 142 232 227 232 F F F <--- 28 560-579 140 140 140 228 228 228 F F F 29 580-599 140 140 140 228 228 228 F F F Max gen: 142, Max flush: 232 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 71 Verifying data now No change after verify Loop: 71 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 141 141 141 229 229 229 F F F 1 020-039 134 134 134 217 217 217 F F F 2 040-059 134 134 134 219 219 219 F F F 3 060-079 141 141 141 229 229 229 F F F 4 080-099 137 137 137 222 222 222 F F F 5 100-119 138 138 138 223 223 223 F F F 6 120-139 142 142 142 232 232 232 F F F 7 140-159 143 143 143 233 233 233 F F F 8 160-179 137 137 137 222 222 222 F F F 9 180-199 142 142 142 230 230 230 F F F 10 200-219 137 137 137 222 222 222 F F F 11 220-239 130 130 130 211 211 211 F F F 12 240-259 142 142 142 230 230 230 F F F 13 260-279 138 138 138 223 223 223 F F F 14 280-299 135 135 135 220 220 220 F F F 15 300-319 142 142 142 232 232 232 F F F 16 320-339 142 142 142 232 232 232 F F F 17 340-359 140 140 140 225 225 225 F F F 18 360-379 142 142 142 231 231 231 F F F 19 380-399 137 137 137 222 222 222 F F F 20 400-419 143 143 143 233 233 233 F F F 21 420-439 134 134 134 218 218 218 F F F 22 440-459 141 141 141 229 229 229 F F F 23 460-479 142 142 142 232 232 232 F F F 24 480-499 143 143 143 233 233 233 F F F 25 500-519 141 141 141 229 229 229 F F F 26 520-539 142 142 142 232 232 232 F F F 27 540-559 142 142 142 232 232 232 F F F 28 560-579 140 140 140 228 228 228 F F F 29 580-599 140 140 140 228 228 228 F F F Max gen: 143, Max flush: 233 Begin loop 72 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:08.632991502Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1709,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:08.640974072Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1709,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:08.641251504Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1709} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:08.641279773Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1709} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:08.641291703Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1709} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:08.641303522Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1709} {"msg":"Crucible fc74a415-aa1d-4113-9050-547943053d76 has session id: 8cfa148f-bd86-43ed-b8d5-7d81f769b806","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:08.641636923Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1709,"session_id":"8cfa148f-bd86-43ed-b8d5-7d81f769b806"} {"msg":"Upstairs opts: Upstairs UUID: fc74a415-aa1d-4113-9050-547943053d76, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:08.641666602Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1709,"session_id":"8cfa148f-bd86-43ed-b8d5-7d81f769b806"} {"msg":"Crucible stats registered with UUID: fc74a415-aa1d-4113-9050-547943053d76","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:08.641677772Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1709,"session_id":"8cfa148f-bd86-43ed-b8d5-7d81f769b806"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:47:08.641686801Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1709,"session_id":"8cfa148f-bd86-43ed-b8d5-7d81f769b806"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:08.641810888Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1709,"":"io task","client":"2","":"downstairs","session_id":"8cfa148f-bd86-43ed-b8d5-7d81f769b806"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:08.641828367Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1709,"":"io task","client":"0","":"downstairs","session_id":"8cfa148f-bd86-43ed-b8d5-7d81f769b806"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:08.641838527Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1709,"":"io task","client":"1","":"downstairs","session_id":"8cfa148f-bd86-43ed-b8d5-7d81f769b806"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:08.641975843Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1709,"task":"crutest"} {"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:08.641990163Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1709} {"msg":"fc74a415-aa1d-4113-9050-547943053d76 active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:08.642039331Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1709,"session_id":"8cfa148f-bd86-43ed-b8d5-7d81f769b806"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:08.642048711Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1709,"client":"0","":"downstairs","session_id":"8cfa148f-bd86-43ed-b8d5-7d81f769b806"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:08.642058531Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1709,"client":"1","":"downstairs","session_id":"8cfa148f-bd86-43ed-b8d5-7d81f769b806"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:08.64206809Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1709,"client":"2","":"downstairs","session_id":"8cfa148f-bd86-43ed-b8d5-7d81f769b806"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:08.64207778Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1709,"":"io task","client":"2","":"downstairs","session_id":"8cfa148f-bd86-43ed-b8d5-7d81f769b806"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:08.64208969Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1709,"":"io task","client":"0","":"downstairs","session_id":"8cfa148f-bd86-43ed-b8d5-7d81f769b806"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:08.642099529Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1709,"":"io task","client":"1","":"downstairs","session_id":"8cfa148f-bd86-43ed-b8d5-7d81f769b806"} {"msg":"ds_connection connected from Ok(127.0.0.1:43424)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:08.642238295Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1709,"":"io task","client":"2","":"downstairs","session_id":"8cfa148f-bd86-43ed-b8d5-7d81f769b806"} {"msg":"ds_connection connected from Ok(127.0.0.1:50543)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:08.642263795Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1709,"":"io task","client":"1","":"downstairs","session_id":"8cfa148f-bd86-43ed-b8d5-7d81f769b806"} {"msg":"ds_connection connected from Ok(127.0.0.1:54570)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:08.642274914Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1709,"":"io task","client":"0","":"downstairs","session_id":"8cfa148f-bd86-43ed-b8d5-7d81f769b806"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:08.642854088Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1709,"client":"0","":"downstairs","session_id":"8cfa148f-bd86-43ed-b8d5-7d81f769b806"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:08.642868057Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1709,"client":"2","":"downstairs","session_id":"8cfa148f-bd86-43ed-b8d5-7d81f769b806"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:08.642921666Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1709,"client":"1","":"downstairs","session_id":"8cfa148f-bd86-43ed-b8d5-7d81f769b806"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:08.643164969Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1709,"session_id":"8cfa148f-bd86-43ed-b8d5-7d81f769b806"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:08.643174248Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1709,"session_id":"8cfa148f-bd86-43ed-b8d5-7d81f769b806"} {"msg":"[0]R flush_numbers[0..12]: [229, 217, 219, 229, 222, 223, 232, 233, 222, 230, 222, 211]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:08.643228627Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1709,"":"downstairs","":"downstairs","session_id":"8cfa148f-bd86-43ed-b8d5-7d81f769b806"} {"msg":"[0]R generation[0..12]: [141, 134, 134, 141, 137, 138, 142, 143, 137, 142, 137, 130]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:08.643239647Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1709,"":"downstairs","":"downstairs","session_id":"8cfa148f-bd86-43ed-b8d5-7d81f769b806"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:08.643250406Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1709,"":"downstairs","":"downstairs","session_id":"8cfa148f-bd86-43ed-b8d5-7d81f769b806"} {"msg":"[1]R flush_numbers[0..12]: [229, 217, 219, 229, 222, 223, 232, 233, 222, 230, 222, 211]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:08.643260966Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1709,"":"downstairs","":"downstairs","session_id":"8cfa148f-bd86-43ed-b8d5-7d81f769b806"} {"msg":"[1]R generation[0..12]: [141, 134, 134, 141, 137, 138, 142, 143, 137, 142, 137, 130]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:08.643274216Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1709,"":"downstairs","":"downstairs","session_id":"8cfa148f-bd86-43ed-b8d5-7d81f769b806"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:08.643285155Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1709,"":"downstairs","":"downstairs","session_id":"8cfa148f-bd86-43ed-b8d5-7d81f769b806"} {"msg":"[2]R flush_numbers[0..12]: [229, 217, 219, 229, 222, 223, 232, 233, 222, 230, 222, 211]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:08.643295775Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1709,"":"downstairs","":"downstairs","session_id":"8cfa148f-bd86-43ed-b8d5-7d81f769b806"} {"msg":"[2]R generation[0..12]: [141, 134, 134, 141, 137, 138, 142, 143, 137, 142, 137, 130]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:08.643306175Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1709,"":"downstairs","":"downstairs","session_id":"8cfa148f-bd86-43ed-b8d5-7d81f769b806"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:08.643316964Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1709,"":"downstairs","":"downstairs","session_id":"8cfa148f-bd86-43ed-b8d5-7d81f769b806"} {"msg":"Max found gen is 144","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:08.643326274Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1709,"":"downstairs","":"downstairs","session_id":"8cfa148f-bd86-43ed-b8d5-7d81f769b806"} {"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:08.643336054Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1709,"":"downstairs","":"downstairs","session_id":"8cfa148f-bd86-43ed-b8d5-7d81f769b806"} {"msg":"Next flush: 234","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:08.643345353Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1709,"":"downstairs","":"downstairs","session_id":"8cfa148f-bd86-43ed-b8d5-7d81f769b806"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:08.643354753Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1709,"":"downstairs","":"downstairs","session_id":"8cfa148f-bd86-43ed-b8d5-7d81f769b806"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:08.643364343Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1709,"session_id":"8cfa148f-bd86-43ed-b8d5-7d81f769b806"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:08.643372963Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1709,"client":"0","":"downstairs","session_id":"8cfa148f-bd86-43ed-b8d5-7d81f769b806"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:08.643382662Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1709,"client":"1","":"downstairs","session_id":"8cfa148f-bd86-43ed-b8d5-7d81f769b806"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:08.643392282Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1709,"client":"2","":"downstairs","session_id":"8cfa148f-bd86-43ed-b8d5-7d81f769b806"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:08.643402002Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1709,"session_id":"8cfa148f-bd86-43ed-b8d5-7d81f769b806"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:08.643410952Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1709,"session_id":"8cfa148f-bd86-43ed-b8d5-7d81f769b806"} {"msg":"fc74a415-aa1d-4113-9050-547943053d76 is now active with session: 8cfa148f-bd86-43ed-b8d5-7d81f769b806","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:08.643421811Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1709,"session_id":"8cfa148f-bd86-43ed-b8d5-7d81f769b806"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:08.643430531Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1709,"session_id":"8cfa148f-bd86-43ed-b8d5-7d81f769b806"} {"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:08.643439551Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1709} {"msg":"Activated sub_volume fc74a415-aa1d-4113-9050-547943053d76","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:08.643447571Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1709} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:08.64345558Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1709,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:08.643753782Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1709,"task":"crutest"} 01/30 Write block 1 len 8192 data: 5 8 02/30 Write block 543 len 28672 data: 13 13 12 10 9 12 13 03/30 Write block 230 len 32768 data: 15 16 11 12 12 9 8 9 04/30 Read block 3 len 12288 05/30 Write block 477 len 32768 data: 16 16 15 13 16 11 10 10 06/30 Read block 8 len 28672 07/30 Read block 268 len 20480 08/30 Write block 232 len 12288 data: 12 13 13 09/30 Write block 489 len 20480 data: 16 17 17 14 12 10/30 Write block 91 len 12288 data: 6 7 9 11/30 Write block 189 len 8192 data: 10 9 12/30 Read block 372 len 4096 13/30 Read block 254 len 20480 14/30 Read block 507 len 16384 15/30 Write block 553 len 8192 data: 13 10 16/30 Read block 199 len 40960 17/30 Read block 551 len 24576 18/30 Flush 19/30 Write block 256 len 28672 data: 13 11 12 12 12 10 11 20/30 Read block 374 len 40960 21/30 Read block 384 len 24576 22/30 Read block 592 len 32768 23/30 Read block 62 len 28672 24/30 Read block 570 len 16384 25/30 Read block 67 len 32768 26/30 Read block 78 len 12288 27/30 Write block 36 len 40960 data: 10 9 7 5 5 6 6 5 4 7 28/30 Write block 333 len 20480 data: 11 11 12 13 14 29/30 Write block 95 len 12288 data: 13 13 13 30/30 Read block 217 len 24576 ---------------------------------------------------------------- Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Sent Done false 1001 Acked Write 7 Done Sent Done false 1002 Acked Write 8 Done Sent Done false 1003 Acked Read 3 Done Sent Done false 1004 Acked Write 8 Done Sent Done false 1005 Acked Read 7 Done Sent Done false 1006 Acked Read 5 Done Sent Done false 1007 Acked Write 3 Done Sent Done false 1008 Acked Write 5 Done Sent Done false 1009 Acked Write 3 Done Sent Done false 1010 Acked Write 2 Done Sent Done false 1011 Acked Read 1 Done Sent Done false 1012 Acked Read 5 Done Sent Done false 1013 Acked Read 4 Done Sent Done false 1014 Acked Write 2 Done Sent Done false 1015 Acked Read 10 Done Sent Done false 1016 Acked Read 6 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Write 7 Done Sent Done false 1019 Acked Read 10 Done Sent Done false 1020 Acked Read 6 Done Sent Done false 1021 Acked Read 8 Done Sent Done false 1022 Acked Read 7 Done Sent Done false 1023 Acked Read 4 Done Sent Done false 1024 Acked Read 8 Done Sent Done false 1025 Acked Read 3 Done Sent Done false 1026 Acked Write 10 Done Sent Done false 1027 Acked Write 5 Done Sent Done false 1028 Acked Write 3 Done Sent Done false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 30 1 31 Done 30 0 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) None Some(JobId(1017)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:47:08.65214135Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1709} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:08.652326244Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1709,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 141 144 234 229 234 F F F <--- 1 020-039 134 134 134 217 217 217 T F T <--- 2 040-059 134 134 134 219 219 219 T F T <--- 3 060-079 141 141 141 229 229 229 F F F 4 080-099 144 137 144 234 222 234 T F T <--- 5 100-119 138 138 138 223 223 223 F F F 6 120-139 142 142 142 232 232 232 F F F 7 140-159 143 143 143 233 233 233 F F F 8 160-179 137 137 137 222 222 222 F F F 9 180-199 144 142 144 234 230 234 F F F <--- 10 200-219 137 137 137 222 222 222 F F F 11 220-239 144 130 144 234 211 234 F F F <--- 12 240-259 142 142 142 230 230 230 T F T <--- 13 260-279 138 138 138 223 223 223 T F T <--- 14 280-299 135 135 135 220 220 220 F F F 15 300-319 142 142 142 232 232 232 F F F 16 320-339 142 142 142 232 232 232 T F T <--- 17 340-359 140 140 140 225 225 225 F F F 18 360-379 142 142 142 231 231 231 F F F 19 380-399 137 137 137 222 222 222 F F F 20 400-419 143 143 143 233 233 233 F F F 21 420-439 134 134 134 218 218 218 F F F 22 440-459 141 141 141 229 229 229 F F F 23 460-479 144 142 144 234 232 234 F F F <--- 24 480-499 144 143 144 234 233 234 F F F <--- 25 500-519 141 141 141 229 229 229 F F F 26 520-539 142 142 142 232 232 232 F F F 27 540-559 144 142 144 234 232 234 F F F <--- 28 560-579 140 140 140 228 228 228 F F F 29 580-599 140 140 140 228 228 228 F F F Max gen: 144, Max flush: 234 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 72 Verifying data now No change after verify Loop: 72 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 234 234 234 F F F 1 020-039 145 145 145 235 235 235 F F F 2 040-059 145 145 145 235 235 235 F F F 3 060-079 141 141 141 229 229 229 F F F 4 080-099 145 145 145 235 235 235 F F F 5 100-119 138 138 138 223 223 223 F F F 6 120-139 142 142 142 232 232 232 F F F 7 140-159 143 143 143 233 233 233 F F F 8 160-179 137 137 137 222 222 222 F F F 9 180-199 144 144 144 234 234 234 F F F 10 200-219 137 137 137 222 222 222 F F F 11 220-239 144 144 144 234 234 234 F F F 12 240-259 145 145 145 235 235 235 F F F 13 260-279 145 145 145 235 235 235 F F F 14 280-299 135 135 135 220 220 220 F F F 15 300-319 142 142 142 232 232 232 F F F 16 320-339 145 145 145 235 235 235 F F F 17 340-359 140 140 140 225 225 225 F F F 18 360-379 142 142 142 231 231 231 F F F 19 380-399 137 137 137 222 222 222 F F F 20 400-419 143 143 143 233 233 233 F F F 21 420-439 134 134 134 218 218 218 F F F 22 440-459 141 141 141 229 229 229 F F F 23 460-479 144 144 144 234 234 234 F F F 24 480-499 144 144 144 234 234 234 F F F 25 500-519 141 141 141 229 229 229 F F F 26 520-539 142 142 142 232 232 232 F F F 27 540-559 144 144 144 234 234 234 F F F 28 560-579 140 140 140 228 228 228 F F F 29 580-599 140 140 140 228 228 228 F F F Max gen: 145, Max flush: 235 Begin loop 73 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:12.900427379Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1717,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:47:12.907481226Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1717,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:12.907533164Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1717,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:12.914323208Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1717,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:12.9145912Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1717} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:12.91461769Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1717} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:12.914626559Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1717} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:12.914638219Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1717} {"msg":"Crucible 59ad34ac-cd21-4018-b711-5d80cde602bb has session id: 02c8d7e1-3c44-4ae3-8b62-621b60918834","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:12.91495338Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1717,"session_id":"02c8d7e1-3c44-4ae3-8b62-621b60918834"} {"msg":"Upstairs opts: Upstairs UUID: 59ad34ac-cd21-4018-b711-5d80cde602bb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:12.914985909Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1717,"session_id":"02c8d7e1-3c44-4ae3-8b62-621b60918834"} {"msg":"Crucible stats registered with UUID: 59ad34ac-cd21-4018-b711-5d80cde602bb","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:12.915004859Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1717,"session_id":"02c8d7e1-3c44-4ae3-8b62-621b60918834"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:47:12.915014348Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1717,"session_id":"02c8d7e1-3c44-4ae3-8b62-621b60918834"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:12.915131695Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1717,"":"io task","client":"0","":"downstairs","session_id":"02c8d7e1-3c44-4ae3-8b62-621b60918834"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:12.915164304Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1717,"":"io task","client":"1","":"downstairs","session_id":"02c8d7e1-3c44-4ae3-8b62-621b60918834"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:12.915175164Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1717,"":"io task","client":"2","":"downstairs","session_id":"02c8d7e1-3c44-4ae3-8b62-621b60918834"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:12.91531412Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1717,"task":"crutest"} {"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:12.915328419Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1717} {"msg":"59ad34ac-cd21-4018-b711-5d80cde602bb active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:12.915360958Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1717,"session_id":"02c8d7e1-3c44-4ae3-8b62-621b60918834"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:12.915370348Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1717,"client":"0","":"downstairs","session_id":"02c8d7e1-3c44-4ae3-8b62-621b60918834"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:12.915380018Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1717,"client":"1","":"downstairs","session_id":"02c8d7e1-3c44-4ae3-8b62-621b60918834"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:12.915389697Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1717,"client":"2","":"downstairs","session_id":"02c8d7e1-3c44-4ae3-8b62-621b60918834"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:12.915399327Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1717,"":"io task","client":"2","":"downstairs","session_id":"02c8d7e1-3c44-4ae3-8b62-621b60918834"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:12.915437266Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1717,"":"io task","client":"0","":"downstairs","session_id":"02c8d7e1-3c44-4ae3-8b62-621b60918834"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:12.915459365Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1717,"":"io task","client":"1","":"downstairs","session_id":"02c8d7e1-3c44-4ae3-8b62-621b60918834"} {"msg":"ds_connection connected from Ok(127.0.0.1:46163)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:12.915504254Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1717,"":"io task","client":"0","":"downstairs","session_id":"02c8d7e1-3c44-4ae3-8b62-621b60918834"} {"msg":"ds_connection connected from Ok(127.0.0.1:63572)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:12.915518034Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1717,"":"io task","client":"2","":"downstairs","session_id":"02c8d7e1-3c44-4ae3-8b62-621b60918834"} {"msg":"ds_connection connected from Ok(127.0.0.1:35563)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:12.915630051Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1717,"":"io task","client":"1","":"downstairs","session_id":"02c8d7e1-3c44-4ae3-8b62-621b60918834"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:12.916110157Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1717,"client":"1","":"downstairs","session_id":"02c8d7e1-3c44-4ae3-8b62-621b60918834"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:12.916190404Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1717,"client":"2","":"downstairs","session_id":"02c8d7e1-3c44-4ae3-8b62-621b60918834"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:12.916202234Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1717,"client":"0","":"downstairs","session_id":"02c8d7e1-3c44-4ae3-8b62-621b60918834"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:12.916450067Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1717,"session_id":"02c8d7e1-3c44-4ae3-8b62-621b60918834"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:12.916483636Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1717,"session_id":"02c8d7e1-3c44-4ae3-8b62-621b60918834"} {"msg":"[0]R flush_numbers[0..12]: [234, 235, 235, 229, 235, 223, 232, 233, 222, 234, 222, 234]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:12.916526355Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1717,"":"downstairs","":"downstairs","session_id":"02c8d7e1-3c44-4ae3-8b62-621b60918834"} {"msg":"[0]R generation[0..12]: [144, 145, 145, 141, 145, 138, 142, 143, 137, 144, 137, 144]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:12.916537804Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1717,"":"downstairs","":"downstairs","session_id":"02c8d7e1-3c44-4ae3-8b62-621b60918834"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:12.916548904Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1717,"":"downstairs","":"downstairs","session_id":"02c8d7e1-3c44-4ae3-8b62-621b60918834"} {"msg":"[1]R flush_numbers[0..12]: [234, 235, 235, 229, 235, 223, 232, 233, 222, 234, 222, 234]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:12.916559474Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1717,"":"downstairs","":"downstairs","session_id":"02c8d7e1-3c44-4ae3-8b62-621b60918834"} {"msg":"[1]R generation[0..12]: [144, 145, 145, 141, 145, 138, 142, 143, 137, 144, 137, 144]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:12.916569913Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1717,"":"downstairs","":"downstairs","session_id":"02c8d7e1-3c44-4ae3-8b62-621b60918834"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:12.916580903Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1717,"":"downstairs","":"downstairs","session_id":"02c8d7e1-3c44-4ae3-8b62-621b60918834"} {"msg":"[2]R flush_numbers[0..12]: [234, 235, 235, 229, 235, 223, 232, 233, 222, 234, 222, 234]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:12.916591443Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1717,"":"downstairs","":"downstairs","session_id":"02c8d7e1-3c44-4ae3-8b62-621b60918834"} {"msg":"[2]R generation[0..12]: [144, 145, 145, 141, 145, 138, 142, 143, 137, 144, 137, 144]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:12.916603942Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1717,"":"downstairs","":"downstairs","session_id":"02c8d7e1-3c44-4ae3-8b62-621b60918834"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:12.916614722Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1717,"":"downstairs","":"downstairs","session_id":"02c8d7e1-3c44-4ae3-8b62-621b60918834"} {"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:12.916624102Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1717,"":"downstairs","":"downstairs","session_id":"02c8d7e1-3c44-4ae3-8b62-621b60918834"} {"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:12.916633742Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1717,"":"downstairs","":"downstairs","session_id":"02c8d7e1-3c44-4ae3-8b62-621b60918834"} {"msg":"Next flush: 236","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:12.916643131Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1717,"":"downstairs","":"downstairs","session_id":"02c8d7e1-3c44-4ae3-8b62-621b60918834"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:12.916652471Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1717,"":"downstairs","":"downstairs","session_id":"02c8d7e1-3c44-4ae3-8b62-621b60918834"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:12.916662071Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1717,"session_id":"02c8d7e1-3c44-4ae3-8b62-621b60918834"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:12.916670771Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1717,"client":"0","":"downstairs","session_id":"02c8d7e1-3c44-4ae3-8b62-621b60918834"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:12.91668042Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1717,"client":"1","":"downstairs","session_id":"02c8d7e1-3c44-4ae3-8b62-621b60918834"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:12.91669007Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1717,"client":"2","":"downstairs","session_id":"02c8d7e1-3c44-4ae3-8b62-621b60918834"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:12.91669976Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1717,"session_id":"02c8d7e1-3c44-4ae3-8b62-621b60918834"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:12.916708809Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1717,"session_id":"02c8d7e1-3c44-4ae3-8b62-621b60918834"} {"msg":"59ad34ac-cd21-4018-b711-5d80cde602bb is now active with session: 02c8d7e1-3c44-4ae3-8b62-621b60918834","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:12.916718729Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1717,"session_id":"02c8d7e1-3c44-4ae3-8b62-621b60918834"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:12.916727419Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1717,"session_id":"02c8d7e1-3c44-4ae3-8b62-621b60918834"} {"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:12.916736439Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1717} {"msg":"Activated sub_volume 59ad34ac-cd21-4018-b711-5d80cde602bb","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:12.916744328Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1717} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:12.916753758Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1717,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:12.917097078Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1717,"task":"crutest"} 01/30 Flush 02/30 Write block 251 len 8192 data: 9 8 03/30 Write block 7 len 16384 data: 13 14 12 12 04/30 Write block 393 len 36864 data: 11 10 13 14 11 9 12 13 13 05/30 Read block 467 len 8192 06/30 Read block 104 len 36864 07/30 Read block 116 len 4096 08/30 Read block 27 len 4096 09/30 Read block 234 len 28672 10/30 Write block 583 len 24576 data: 14 13 13 14 18 16 11/30 Write block 393 len 20480 data: 12 11 14 15 12 12/30 Flush 13/30 Flush 14/30 Write block 186 len 4096 data: 13 15/30 Write block 549 len 8192 data: 14 12 16/30 Read block 236 len 24576 17/30 Write block 112 len 8192 data: 12 12 18/30 Flush 19/30 Write block 501 len 32768 data: 15 15 16 17 16 13 15 9 20/30 Write block 376 len 32768 data: 15 16 17 17 17 17 20 14 21/30 Write block 74 len 4096 data: 14 22/30 Read block 577 len 8192 23/30 Write block 9 len 8192 data: 13 13 24/30 Read block 558 len 32768 25/30 Write block 51 len 36864 data: 10 9 8 8 11 11 11 14 12 26/30 Write block 241 len 32768 data: 11 9 9 10 10 12 14 14 27/30 Read block 77 len 8192 28/30 Read block 161 len 16384 29/30 Read block 139 len 4096 30/30 Read block 314 len 12288 ---------------------------------------------------------------- Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 2 Done Done Done false 1002 Acked Write 4 Done Done Done false 1003 Acked Write 9 Done Done Done false 1004 Acked Read 2 Done Done Done false 1005 Acked Read 9 Sent Done Done false 1006 Acked Read 1 Sent Done Done false 1007 Acked Read 1 Sent Done Done false 1008 Acked Read 7 Sent Done Done false 1009 Acked Write 6 Sent Done Done false 1010 Acked Write 5 Sent Done Done false 1011 Acked Flush 0 Sent Done Done false 1012 Acked Flush 0 Sent Done Done false 1013 Acked Write 1 Sent Done Done false 1014 Acked Write 2 Sent Done Done false 1015 Acked Read 6 Sent Done Done false 1016 Acked Write 2 Sent Done Done false 1017 Acked Flush 0 Sent Done Done false 1018 Acked Write 8 Sent Done Done false 1019 Acked Write 8 Sent Done Done false 1020 Acked Write 1 Sent Done Done false 1021 Acked Read 2 Sent Done Done false 1022 Acked Write 2 Sent Done Done false 1023 Acked Read 8 Sent Done Done false 1024 Acked Write 9 Sent Done Done false 1025 Acked Write 8 Sent Done Done false 1026 Acked Read 2 Sent Done Done false 1027 Acked Read 4 Sent Done Done false 1028 Acked Read 1 Sent Done Done false 1029 Acked Read 3 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 25 1 0 26 Done 4 28 29 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1000)) Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:47:12.924542803Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1717} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:12.924769517Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1717,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 146 146 234 237 237 T T T <--- 1 020-039 145 145 145 235 235 235 F F F 2 040-059 145 145 145 235 235 235 F T T <--- 3 060-079 141 141 141 229 229 229 F T T <--- 4 080-099 145 145 145 235 235 235 F F F 5 100-119 138 146 146 223 239 239 F F F <--- 6 120-139 142 142 142 232 232 232 F F F 7 140-159 143 143 143 233 233 233 F F F 8 160-179 137 137 137 222 222 222 F F F 9 180-199 144 146 146 234 239 239 F F F <--- 10 200-219 137 137 137 222 222 222 F F F 11 220-239 144 144 144 234 234 234 F F F 12 240-259 145 146 146 235 237 237 T T T <--- 13 260-279 145 145 145 235 235 235 F F F 14 280-299 135 135 135 220 220 220 F F F 15 300-319 142 142 142 232 232 232 F F F 16 320-339 145 145 145 235 235 235 F F F 17 340-359 140 140 140 225 225 225 F F F 18 360-379 142 142 142 231 231 231 F T T <--- 19 380-399 137 146 146 222 237 237 T T T <--- 20 400-419 143 146 146 233 237 237 T F F <--- 21 420-439 134 134 134 218 218 218 F F F 22 440-459 141 141 141 229 229 229 F F F 23 460-479 144 144 144 234 234 234 F F F 24 480-499 144 144 144 234 234 234 F F F 25 500-519 141 141 141 229 229 229 F T T <--- 26 520-539 142 142 142 232 232 232 F F F 27 540-559 144 146 146 234 239 239 F F F <--- 28 560-579 140 140 140 228 228 228 F F F 29 580-599 140 146 146 228 237 237 F F F <--- Max gen: 146, Max flush: 239 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 73 Verifying data now No change after verify Loop: 73 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 147 147 147 240 240 240 F F F 1 020-039 145 145 145 235 235 235 F F F 2 040-059 147 147 147 240 240 240 F F F 3 060-079 147 147 147 240 240 240 F F F 4 080-099 145 145 145 235 235 235 F F F 5 100-119 146 146 146 239 239 239 F F F 6 120-139 142 142 142 232 232 232 F F F 7 140-159 143 143 143 233 233 233 F F F 8 160-179 137 137 137 222 222 222 F F F 9 180-199 146 146 146 239 239 239 F F F 10 200-219 137 137 137 222 222 222 F F F 11 220-239 144 144 144 234 234 234 F F F 12 240-259 147 147 147 240 240 240 F F F 13 260-279 145 145 145 235 235 235 F F F 14 280-299 135 135 135 220 220 220 F F F 15 300-319 142 142 142 232 232 232 F F F 16 320-339 145 145 145 235 235 235 F F F 17 340-359 140 140 140 225 225 225 F F F 18 360-379 147 147 147 240 240 240 F F F 19 380-399 147 147 147 240 240 240 F F F 20 400-419 146 146 146 237 237 237 F F F 21 420-439 134 134 134 218 218 218 F F F 22 440-459 141 141 141 229 229 229 F F F 23 460-479 144 144 144 234 234 234 F F F 24 480-499 144 144 144 234 234 234 F F F 25 500-519 147 147 147 240 240 240 F F F 26 520-539 142 142 142 232 232 232 F F F 27 540-559 146 146 146 239 239 239 F F F 28 560-579 140 140 140 228 228 228 F F F 29 580-599 146 146 146 237 237 237 F F F Max gen: 147, Max flush: 240 Begin loop 74 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:17.177327913Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1725,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:47:17.184310872Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1725,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:17.184358721Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1725,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:17.190995739Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1725,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:17.191257782Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1725} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:17.191285831Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1725} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:17.191294441Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1725} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:17.19130587Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1725} {"msg":"Crucible dd269bd1-83ee-40a7-894f-c8451ec3ce8b has session id: 231a31e4-d415-43b2-8e37-318f3a8ab097","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:17.191612971Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1725,"session_id":"231a31e4-d415-43b2-8e37-318f3a8ab097"} {"msg":"Upstairs opts: Upstairs UUID: dd269bd1-83ee-40a7-894f-c8451ec3ce8b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:17.19166243Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1725,"session_id":"231a31e4-d415-43b2-8e37-318f3a8ab097"} {"msg":"Crucible stats registered with UUID: dd269bd1-83ee-40a7-894f-c8451ec3ce8b","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:17.1916729Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1725,"session_id":"231a31e4-d415-43b2-8e37-318f3a8ab097"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:47:17.191682309Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1725,"session_id":"231a31e4-d415-43b2-8e37-318f3a8ab097"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:17.191809896Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1725,"":"io task","client":"2","":"downstairs","session_id":"231a31e4-d415-43b2-8e37-318f3a8ab097"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:17.191828975Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1725,"":"io task","client":"0","":"downstairs","session_id":"231a31e4-d415-43b2-8e37-318f3a8ab097"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:17.191839145Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1725,"":"io task","client":"1","":"downstairs","session_id":"231a31e4-d415-43b2-8e37-318f3a8ab097"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:17.191972231Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1725,"task":"crutest"} {"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:17.19199681Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1725} {"msg":"dd269bd1-83ee-40a7-894f-c8451ec3ce8b active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:17.19201998Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1725,"session_id":"231a31e4-d415-43b2-8e37-318f3a8ab097"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:17.192034149Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1725,"client":"0","":"downstairs","session_id":"231a31e4-d415-43b2-8e37-318f3a8ab097"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:17.192043929Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1725,"client":"1","":"downstairs","session_id":"231a31e4-d415-43b2-8e37-318f3a8ab097"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:17.192053599Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1725,"client":"2","":"downstairs","session_id":"231a31e4-d415-43b2-8e37-318f3a8ab097"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:17.192063108Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1725,"":"io task","client":"2","":"downstairs","session_id":"231a31e4-d415-43b2-8e37-318f3a8ab097"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:17.192073138Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1725,"":"io task","client":"0","":"downstairs","session_id":"231a31e4-d415-43b2-8e37-318f3a8ab097"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:17.192083048Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1725,"":"io task","client":"1","":"downstairs","session_id":"231a31e4-d415-43b2-8e37-318f3a8ab097"} {"msg":"ds_connection connected from Ok(127.0.0.1:46740)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:17.192161026Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1725,"":"io task","client":"1","":"downstairs","session_id":"231a31e4-d415-43b2-8e37-318f3a8ab097"} {"msg":"ds_connection connected from Ok(127.0.0.1:41350)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:17.192187505Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1725,"":"io task","client":"2","":"downstairs","session_id":"231a31e4-d415-43b2-8e37-318f3a8ab097"} {"msg":"ds_connection connected from Ok(127.0.0.1:41029)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:17.192198744Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1725,"":"io task","client":"0","":"downstairs","session_id":"231a31e4-d415-43b2-8e37-318f3a8ab097"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:17.192903574Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1725,"client":"1","":"downstairs","session_id":"231a31e4-d415-43b2-8e37-318f3a8ab097"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:17.192939563Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1725,"client":"2","":"downstairs","session_id":"231a31e4-d415-43b2-8e37-318f3a8ab097"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:17.193083249Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1725,"client":"0","":"downstairs","session_id":"231a31e4-d415-43b2-8e37-318f3a8ab097"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:17.193229055Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1725,"session_id":"231a31e4-d415-43b2-8e37-318f3a8ab097"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:17.193244624Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1725,"session_id":"231a31e4-d415-43b2-8e37-318f3a8ab097"} {"msg":"[0]R flush_numbers[0..12]: [240, 235, 240, 240, 235, 239, 232, 233, 222, 239, 222, 234]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:17.19339565Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1725,"":"downstairs","":"downstairs","session_id":"231a31e4-d415-43b2-8e37-318f3a8ab097"} {"msg":"[0]R generation[0..12]: [147, 145, 147, 147, 145, 146, 142, 143, 137, 146, 137, 144]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:17.19340716Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1725,"":"downstairs","":"downstairs","session_id":"231a31e4-d415-43b2-8e37-318f3a8ab097"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:17.193418159Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1725,"":"downstairs","":"downstairs","session_id":"231a31e4-d415-43b2-8e37-318f3a8ab097"} {"msg":"[1]R flush_numbers[0..12]: [240, 235, 240, 240, 235, 239, 232, 233, 222, 239, 222, 234]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:17.193428859Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1725,"":"downstairs","":"downstairs","session_id":"231a31e4-d415-43b2-8e37-318f3a8ab097"} {"msg":"[1]R generation[0..12]: [147, 145, 147, 147, 145, 146, 142, 143, 137, 146, 137, 144]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:17.193439359Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1725,"":"downstairs","":"downstairs","session_id":"231a31e4-d415-43b2-8e37-318f3a8ab097"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:17.193450188Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1725,"":"downstairs","":"downstairs","session_id":"231a31e4-d415-43b2-8e37-318f3a8ab097"} {"msg":"[2]R flush_numbers[0..12]: [240, 235, 240, 240, 235, 239, 232, 233, 222, 239, 222, 234]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:17.193460768Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1725,"":"downstairs","":"downstairs","session_id":"231a31e4-d415-43b2-8e37-318f3a8ab097"} {"msg":"[2]R generation[0..12]: [147, 145, 147, 147, 145, 146, 142, 143, 137, 146, 137, 144]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:17.193471328Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1725,"":"downstairs","":"downstairs","session_id":"231a31e4-d415-43b2-8e37-318f3a8ab097"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:17.193482167Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1725,"":"downstairs","":"downstairs","session_id":"231a31e4-d415-43b2-8e37-318f3a8ab097"} {"msg":"Max found gen is 148","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:17.193491637Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1725,"":"downstairs","":"downstairs","session_id":"231a31e4-d415-43b2-8e37-318f3a8ab097"} {"msg":"Generation requested: 148 >= found:148","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:17.193503447Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1725,"":"downstairs","":"downstairs","session_id":"231a31e4-d415-43b2-8e37-318f3a8ab097"} {"msg":"Next flush: 241","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:17.193512707Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1725,"":"downstairs","":"downstairs","session_id":"231a31e4-d415-43b2-8e37-318f3a8ab097"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:17.193521966Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1725,"":"downstairs","":"downstairs","session_id":"231a31e4-d415-43b2-8e37-318f3a8ab097"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:17.193531826Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1725,"session_id":"231a31e4-d415-43b2-8e37-318f3a8ab097"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:17.193540356Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1725,"client":"0","":"downstairs","session_id":"231a31e4-d415-43b2-8e37-318f3a8ab097"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:17.193549856Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1725,"client":"1","":"downstairs","session_id":"231a31e4-d415-43b2-8e37-318f3a8ab097"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:17.193559345Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1725,"client":"2","":"downstairs","session_id":"231a31e4-d415-43b2-8e37-318f3a8ab097"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:17.193569185Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1725,"session_id":"231a31e4-d415-43b2-8e37-318f3a8ab097"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:17.193578255Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1725,"session_id":"231a31e4-d415-43b2-8e37-318f3a8ab097"} {"msg":"dd269bd1-83ee-40a7-894f-c8451ec3ce8b is now active with session: 231a31e4-d415-43b2-8e37-318f3a8ab097","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:17.193588054Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1725,"session_id":"231a31e4-d415-43b2-8e37-318f3a8ab097"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:17.193596704Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1725,"session_id":"231a31e4-d415-43b2-8e37-318f3a8ab097"} {"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-08-03T22:47:17.193605694Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1725} {"msg":"Activated sub_volume dd269bd1-83ee-40a7-894f-c8451ec3ce8b","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:17.193619144Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1725} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:17.193627203Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1725,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:17.193877406Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1725,"task":"crutest"} 01/30 Write block 501 len 4096 data: 16 02/30 Read block 23 len 28672 03/30 Write block 325 len 28672 data: 17 18 19 15 14 12 13 04/30 Read block 538 len 12288 05/30 Flush 06/30 Write block 146 len 36864 data: 15 14 12 8 8 9 8 7 9 07/30 Read block 441 len 28672 08/30 Flush 09/30 Write block 576 len 16384 data: 13 11 12 11 10/30 Flush 11/30 Write block 242 len 40960 data: 10 10 11 11 13 15 15 12 12 10 12/30 Flush 13/30 Write block 235 len 36864 data: 10 9 10 8 9 9 12 11 11 14/30 Write block 360 len 20480 data: 11 13 16 17 17 15/30 Write block 448 len 8192 data: 7 9 16/30 Read block 112 len 32768 17/30 Read block 78 len 36864 18/30 Read block 447 len 16384 19/30 Read block 585 len 24576 20/30 Read block 492 len 20480 21/30 Read block 139 len 20480 22/30 Read block 116 len 4096 23/30 Read block 385 len 16384 24/30 Flush 25/30 Write block 202 len 4096 data: 18 26/30 Flush 27/30 Write block 399 len 36864 data: 13 14 14 15 13 14 14 11 10 28/30 Read block 530 len 12288 29/30 Read block 75 len 12288 30/30 Write block 442 len 20480 data: 8 7 9 8 8 ---------------------------------------------------------------- Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 9 Done Done Done false 1013 Acked Write 5 Done Done Done false 1014 Acked Write 2 Done Done Done false 1015 Acked Read 8 Done Done Done false 1016 Acked Read 9 Sent Done Done false 1017 Acked Read 4 Sent Done Done false 1018 Acked Read 6 Sent Done Done false 1019 Acked Read 5 Sent Done Done false 1020 Acked Read 5 Sent Done Done false 1021 Acked Read 1 Sent Done Done false 1022 Acked Read 4 Sent Done Done false 1023 Acked Flush 0 Sent Done Done false 1024 Acked Write 1 Sent Done Done false 1025 Acked Flush 0 Sent Done Done false 1026 Acked Write 9 Sent Done Done false 1027 Acked Read 3 Sent Done Done false 1028 Acked Read 3 Sent Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 14 1 1 16 Done 4 17 17 38 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1011)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:47:17.203176508Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1725} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:17.203356083Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1725,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 147 147 147 240 240 240 F F F 1 020-039 145 145 145 235 235 235 F F F 2 040-059 147 147 147 240 240 240 F F F 3 060-079 147 147 147 240 240 240 F F F 4 080-099 145 145 145 235 235 235 F F F 5 100-119 146 146 146 239 239 239 F F F 6 120-139 142 142 142 232 232 232 F F F 7 140-159 148 148 148 242 242 242 F F F 8 160-179 137 137 137 222 222 222 F F F 9 180-199 146 146 146 239 239 239 F F F 10 200-219 137 148 148 222 246 246 F F F <--- 11 220-239 144 148 148 234 245 245 T F F <--- 12 240-259 148 148 148 244 245 245 T F F <--- 13 260-279 145 145 145 235 235 235 F F F 14 280-299 135 135 135 220 220 220 F F F 15 300-319 142 142 142 232 232 232 F F F 16 320-339 148 148 148 241 241 241 F F F 17 340-359 140 140 140 225 225 225 F F F 18 360-379 147 148 148 240 245 245 T F F <--- 19 380-399 147 147 147 240 240 240 F T T <--- 20 400-419 146 146 146 237 237 237 F T T <--- 21 420-439 134 134 134 218 218 218 F F F 22 440-459 141 148 148 229 245 245 T T T <--- 23 460-479 144 144 144 234 234 234 F F F 24 480-499 144 144 144 234 234 234 F F F 25 500-519 148 148 148 241 241 241 F F F 26 520-539 142 142 142 232 232 232 F F F 27 540-559 146 146 146 239 239 239 F F F 28 560-579 148 148 148 243 243 243 F F F 29 580-599 146 146 146 237 237 237 F F F Max gen: 148, Max flush: 246 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 74 Verifying data now No change after verify Loop: 74 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 147 147 147 240 240 240 F F F 1 020-039 145 145 145 235 235 235 F F F 2 040-059 147 147 147 240 240 240 F F F 3 060-079 147 147 147 240 240 240 F F F 4 080-099 145 145 145 235 235 235 F F F 5 100-119 146 146 146 239 239 239 F F F 6 120-139 142 142 142 232 232 232 F F F 7 140-159 148 148 148 242 242 242 F F F 8 160-179 137 137 137 222 222 222 F F F 9 180-199 146 146 146 239 239 239 F F F 10 200-219 148 148 148 246 246 246 F F F 11 220-239 148 148 148 245 245 245 F F F 12 240-259 148 148 148 245 245 245 F F F 13 260-279 145 145 145 235 235 235 F F F 14 280-299 135 135 135 220 220 220 F F F 15 300-319 142 142 142 232 232 232 F F F 16 320-339 148 148 148 241 241 241 F F F 17 340-359 140 140 140 225 225 225 F F F 18 360-379 148 148 148 245 245 245 F F F 19 380-399 149 149 149 247 247 247 F F F 20 400-419 149 149 149 247 247 247 F F F 21 420-439 134 134 134 218 218 218 F F F 22 440-459 149 149 149 247 247 247 F F F 23 460-479 144 144 144 234 234 234 F F F 24 480-499 144 144 144 234 234 234 F F F 25 500-519 148 148 148 241 241 241 F F F 26 520-539 142 142 142 232 232 232 F F F 27 540-559 146 146 146 239 239 239 F F F 28 560-579 148 148 148 243 243 243 F F F 29 580-599 146 146 146 237 237 237 F F F Max gen: 149, Max flush: 247 Begin loop 75 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:21.446391984Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1733,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:21.454579318Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1733,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:21.454873319Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1733} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:21.454900788Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1733} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:21.454911458Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1733} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:21.454923018Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1733} {"msg":"Crucible 877397d1-13c8-43e2-a5fc-284b13fed56f has session id: e035f87d-d42d-447a-906b-1ef9814aa3f0","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:21.45519712Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1733,"session_id":"e035f87d-d42d-447a-906b-1ef9814aa3f0"} {"msg":"Upstairs opts: Upstairs UUID: 877397d1-13c8-43e2-a5fc-284b13fed56f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:21.455271868Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1733,"session_id":"e035f87d-d42d-447a-906b-1ef9814aa3f0"} {"msg":"Crucible stats registered with UUID: 877397d1-13c8-43e2-a5fc-284b13fed56f","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:21.455282237Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1733,"session_id":"e035f87d-d42d-447a-906b-1ef9814aa3f0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:47:21.455291217Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1733,"session_id":"e035f87d-d42d-447a-906b-1ef9814aa3f0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:21.455436663Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1733,"":"io task","client":"1","":"downstairs","session_id":"e035f87d-d42d-447a-906b-1ef9814aa3f0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:21.455469022Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1733,"":"io task","client":"2","":"downstairs","session_id":"e035f87d-d42d-447a-906b-1ef9814aa3f0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:21.455479852Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1733,"":"io task","client":"0","":"downstairs","session_id":"e035f87d-d42d-447a-906b-1ef9814aa3f0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:21.455597068Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1733,"task":"crutest"} {"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:21.455608678Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1733} {"msg":"877397d1-13c8-43e2-a5fc-284b13fed56f active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:21.455625008Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1733,"session_id":"e035f87d-d42d-447a-906b-1ef9814aa3f0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:21.455633887Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1733,"client":"0","":"downstairs","session_id":"e035f87d-d42d-447a-906b-1ef9814aa3f0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:21.455643617Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1733,"client":"1","":"downstairs","session_id":"e035f87d-d42d-447a-906b-1ef9814aa3f0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:21.455653217Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1733,"client":"2","":"downstairs","session_id":"e035f87d-d42d-447a-906b-1ef9814aa3f0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:21.455662856Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1733,"":"io task","client":"2","":"downstairs","session_id":"e035f87d-d42d-447a-906b-1ef9814aa3f0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:21.455674786Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1733,"":"io task","client":"0","":"downstairs","session_id":"e035f87d-d42d-447a-906b-1ef9814aa3f0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:21.455699805Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1733,"":"io task","client":"1","":"downstairs","session_id":"e035f87d-d42d-447a-906b-1ef9814aa3f0"} {"msg":"ds_connection connected from Ok(127.0.0.1:56748)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:21.455807392Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1733,"":"io task","client":"2","":"downstairs","session_id":"e035f87d-d42d-447a-906b-1ef9814aa3f0"} {"msg":"ds_connection connected from Ok(127.0.0.1:52963)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:21.455833872Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1733,"":"io task","client":"0","":"downstairs","session_id":"e035f87d-d42d-447a-906b-1ef9814aa3f0"} {"msg":"ds_connection connected from Ok(127.0.0.1:37704)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:21.455845171Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1733,"":"io task","client":"1","":"downstairs","session_id":"e035f87d-d42d-447a-906b-1ef9814aa3f0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:21.456467493Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1733,"client":"0","":"downstairs","session_id":"e035f87d-d42d-447a-906b-1ef9814aa3f0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:21.456486723Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1733,"client":"1","":"downstairs","session_id":"e035f87d-d42d-447a-906b-1ef9814aa3f0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:21.45657974Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1733,"client":"2","":"downstairs","session_id":"e035f87d-d42d-447a-906b-1ef9814aa3f0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:21.456908911Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1733,"session_id":"e035f87d-d42d-447a-906b-1ef9814aa3f0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:21.45692168Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1733,"session_id":"e035f87d-d42d-447a-906b-1ef9814aa3f0"} {"msg":"[0]R flush_numbers[0..12]: [240, 235, 240, 240, 235, 239, 232, 242, 222, 239, 246, 245]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:21.456955919Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1733,"":"downstairs","":"downstairs","session_id":"e035f87d-d42d-447a-906b-1ef9814aa3f0"} {"msg":"[0]R generation[0..12]: [147, 145, 147, 147, 145, 146, 142, 148, 137, 146, 148, 148]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:21.456967209Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1733,"":"downstairs","":"downstairs","session_id":"e035f87d-d42d-447a-906b-1ef9814aa3f0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:21.456978199Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1733,"":"downstairs","":"downstairs","session_id":"e035f87d-d42d-447a-906b-1ef9814aa3f0"} {"msg":"[1]R flush_numbers[0..12]: [240, 235, 240, 240, 235, 239, 232, 242, 222, 239, 246, 245]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:21.456988718Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1733,"":"downstairs","":"downstairs","session_id":"e035f87d-d42d-447a-906b-1ef9814aa3f0"} {"msg":"[1]R generation[0..12]: [147, 145, 147, 147, 145, 146, 142, 148, 137, 146, 148, 148]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:21.457002058Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1733,"":"downstairs","":"downstairs","session_id":"e035f87d-d42d-447a-906b-1ef9814aa3f0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:21.457013148Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1733,"":"downstairs","":"downstairs","session_id":"e035f87d-d42d-447a-906b-1ef9814aa3f0"} {"msg":"[2]R flush_numbers[0..12]: [240, 235, 240, 240, 235, 239, 232, 242, 222, 239, 246, 245]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:21.457023657Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1733,"":"downstairs","":"downstairs","session_id":"e035f87d-d42d-447a-906b-1ef9814aa3f0"} {"msg":"[2]R generation[0..12]: [147, 145, 147, 147, 145, 146, 142, 148, 137, 146, 148, 148]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:21.457034197Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1733,"":"downstairs","":"downstairs","session_id":"e035f87d-d42d-447a-906b-1ef9814aa3f0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:21.457045027Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1733,"":"downstairs","":"downstairs","session_id":"e035f87d-d42d-447a-906b-1ef9814aa3f0"} {"msg":"Max found gen is 150","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:21.457054396Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1733,"":"downstairs","":"downstairs","session_id":"e035f87d-d42d-447a-906b-1ef9814aa3f0"} {"msg":"Generation requested: 150 >= found:150","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:21.457064056Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1733,"":"downstairs","":"downstairs","session_id":"e035f87d-d42d-447a-906b-1ef9814aa3f0"} {"msg":"Next flush: 248","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:21.457073326Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1733,"":"downstairs","":"downstairs","session_id":"e035f87d-d42d-447a-906b-1ef9814aa3f0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:21.457082726Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1733,"":"downstairs","":"downstairs","session_id":"e035f87d-d42d-447a-906b-1ef9814aa3f0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:21.457092675Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1733,"session_id":"e035f87d-d42d-447a-906b-1ef9814aa3f0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:21.457101525Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1733,"client":"0","":"downstairs","session_id":"e035f87d-d42d-447a-906b-1ef9814aa3f0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:21.457111435Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1733,"client":"1","":"downstairs","session_id":"e035f87d-d42d-447a-906b-1ef9814aa3f0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:21.457121214Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1733,"client":"2","":"downstairs","session_id":"e035f87d-d42d-447a-906b-1ef9814aa3f0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:21.457131224Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1733,"session_id":"e035f87d-d42d-447a-906b-1ef9814aa3f0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:21.457140504Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1733,"session_id":"e035f87d-d42d-447a-906b-1ef9814aa3f0"} {"msg":"877397d1-13c8-43e2-a5fc-284b13fed56f is now active with session: e035f87d-d42d-447a-906b-1ef9814aa3f0","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:21.457152024Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1733,"session_id":"e035f87d-d42d-447a-906b-1ef9814aa3f0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:21.457160693Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1733,"session_id":"e035f87d-d42d-447a-906b-1ef9814aa3f0"} {"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:21.457169673Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1733} {"msg":"Activated sub_volume 877397d1-13c8-43e2-a5fc-284b13fed56f","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:21.457177713Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1733} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:21.457185733Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1733,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:21.457495144Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1733,"task":"crutest"} 01/30 Write block 125 len 20480 data: 12 11 12 14 12 02/30 Read block 110 len 24576 03/30 Write block 377 len 12288 data: 17 18 18 04/30 Read block 323 len 36864 05/30 Read block 251 len 40960 06/30 Write block 173 len 4096 data: 12 07/30 Read block 403 len 24576 08/30 Flush 09/30 Write block 50 len 32768 data: 12 11 10 9 9 12 12 12 10/30 Write block 280 len 24576 data: 11 11 7 8 9 12 11/30 Read block 133 len 40960 12/30 Read block 296 len 16384 13/30 Read block 198 len 32768 14/30 Write block 531 len 8192 data: 8 14 15/30 Read block 296 len 12288 16/30 Write block 379 len 40960 data: 19 18 18 21 15 13 12 10 11 11 17/30 Read block 172 len 20480 18/30 Read block 358 len 40960 19/30 Read block 233 len 12288 20/30 Read block 217 len 24576 21/30 Read block 538 len 36864 22/30 Read block 437 len 36864 23/30 Read block 234 len 28672 24/30 Read block 115 len 32768 25/30 Write block 475 len 32768 data: 12 17 17 17 16 14 17 12 26/30 Read block 318 len 32768 27/30 Read block 482 len 40960 28/30 Read block 229 len 4096 29/30 Read block 406 len 28672 30/30 Write block 278 len 16384 data: 10 11 12 12 ---------------------------------------------------------------- Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Read 6 Done Done Done false 1002 Acked Write 3 Done Done Done false 1003 Acked Read 9 Done Sent Done false 1004 Acked Read 10 Done Sent Done false 1005 Acked Write 1 Done Sent Done false 1006 Acked Read 6 Done Sent Done false 1007 Acked Flush 0 Done Sent Done false 1008 Acked Write 8 Done Sent Done false 1009 Acked Write 6 Done Sent Done false 1010 Acked Read 10 Done Sent Done false 1011 Acked Read 4 Done Sent Done false 1012 Acked Read 8 Done Sent Done false 1013 Acked Write 2 Done Sent Done false 1014 Acked Read 3 Done Sent Done false 1015 Acked Write 10 Done Sent Done false 1016 Acked Read 5 Done Sent Done false 1017 Acked Read 10 Done Sent Done false 1018 Acked Read 3 Done Sent Done false 1019 Acked Read 6 Done Sent Done false 1020 Acked Read 9 Done Sent Done false 1021 Acked Read 9 Done Sent Done false 1022 Acked Read 7 Done Sent Done false 1023 Acked Read 8 Done Sent Done false 1024 Acked Write 8 Done Sent Done false 1025 Acked Read 8 Done Sent Done false 1026 Acked Read 10 Done Sent Done false 1027 Acked Read 1 Done Sent Done false 1028 Acked Read 7 Done Sent Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 27 1 29 Done 29 3 29 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1007)) None Some(JobId(1007)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:47:21.467391608Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1733} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:21.467564413Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1733,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 147 147 147 240 240 240 F F F 1 020-039 145 145 145 235 235 235 F F F 2 040-059 147 147 147 240 240 240 T F T <--- 3 060-079 147 147 147 240 240 240 F F F 4 080-099 145 145 145 235 235 235 F F F 5 100-119 146 146 146 239 239 239 F F F 6 120-139 150 142 150 248 232 248 F T F <--- 7 140-159 148 148 148 242 242 242 F F F 8 160-179 150 137 150 248 222 248 F F F <--- 9 180-199 146 146 146 239 239 239 F F F 10 200-219 148 148 148 246 246 246 F F F 11 220-239 148 148 148 245 245 245 F F F 12 240-259 148 148 148 245 245 245 F F F 13 260-279 145 145 145 235 235 235 T F T <--- 14 280-299 135 135 135 220 220 220 T F T <--- 15 300-319 142 142 142 232 232 232 F F F 16 320-339 148 148 148 241 241 241 F F F 17 340-359 140 140 140 225 225 225 F F F 18 360-379 150 148 150 248 245 248 T T T <--- 19 380-399 149 149 149 247 247 247 T F T <--- 20 400-419 149 149 149 247 247 247 F F F 21 420-439 134 134 134 218 218 218 F F F 22 440-459 149 149 149 247 247 247 F F F 23 460-479 144 144 144 234 234 234 T F T <--- 24 480-499 144 144 144 234 234 234 T F T <--- 25 500-519 148 148 148 241 241 241 F F F 26 520-539 142 142 142 232 232 232 T F T <--- 27 540-559 146 146 146 239 239 239 F F F 28 560-579 148 148 148 243 243 243 F F F 29 580-599 146 146 146 237 237 237 F F F Max gen: 150, Max flush: 248 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 75 Verifying data now No change after verify Loop: 75 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 147 147 147 240 240 240 F F F 1 020-039 145 145 145 235 235 235 F F F 2 040-059 151 151 151 249 249 249 F F F 3 060-079 147 147 147 240 240 240 F F F 4 080-099 145 145 145 235 235 235 F F F 5 100-119 146 146 146 239 239 239 F F F 6 120-139 150 150 150 248 248 248 F F F 7 140-159 148 148 148 242 242 242 F F F 8 160-179 150 150 150 248 248 248 F F F 9 180-199 146 146 146 239 239 239 F F F 10 200-219 148 148 148 246 246 246 F F F 11 220-239 148 148 148 245 245 245 F F F 12 240-259 148 148 148 245 245 245 F F F 13 260-279 151 151 151 249 249 249 F F F 14 280-299 151 151 151 249 249 249 F F F 15 300-319 142 142 142 232 232 232 F F F 16 320-339 148 148 148 241 241 241 F F F 17 340-359 140 140 140 225 225 225 F F F 18 360-379 151 151 151 249 249 249 F F F 19 380-399 151 151 151 249 249 249 F F F 20 400-419 149 149 149 247 247 247 F F F 21 420-439 134 134 134 218 218 218 F F F 22 440-459 149 149 149 247 247 247 F F F 23 460-479 151 151 151 249 249 249 F F F 24 480-499 151 151 151 249 249 249 F F F 25 500-519 148 148 148 241 241 241 F F F 26 520-539 151 151 151 249 249 249 F F F 27 540-559 146 146 146 239 239 239 F F F 28 560-579 148 148 148 243 243 243 F F F 29 580-599 146 146 146 237 237 237 F F F Max gen: 151, Max flush: 249 Begin loop 76 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:25.713278657Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1741,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:25.721266437Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1741,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:25.721585998Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1741} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:25.721621566Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1741} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:25.721631006Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1741} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:25.721642706Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1741} {"msg":"Crucible 4dc33c9a-5278-471e-8b9b-9156f53b9928 has session id: 3c2f5a8d-cec2-404e-8c9f-c94ae1986c50","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:25.721939797Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1741,"session_id":"3c2f5a8d-cec2-404e-8c9f-c94ae1986c50"} {"msg":"Upstairs opts: Upstairs UUID: 4dc33c9a-5278-471e-8b9b-9156f53b9928, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:25.721972476Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1741,"session_id":"3c2f5a8d-cec2-404e-8c9f-c94ae1986c50"} {"msg":"Crucible stats registered with UUID: 4dc33c9a-5278-471e-8b9b-9156f53b9928","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:25.722021855Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1741,"session_id":"3c2f5a8d-cec2-404e-8c9f-c94ae1986c50"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:47:25.722047394Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1741,"session_id":"3c2f5a8d-cec2-404e-8c9f-c94ae1986c50"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:25.722163651Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1741,"":"io task","client":"2","":"downstairs","session_id":"3c2f5a8d-cec2-404e-8c9f-c94ae1986c50"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:25.72218218Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1741,"":"io task","client":"0","":"downstairs","session_id":"3c2f5a8d-cec2-404e-8c9f-c94ae1986c50"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:25.72219243Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1741,"":"io task","client":"1","":"downstairs","session_id":"3c2f5a8d-cec2-404e-8c9f-c94ae1986c50"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:25.722312127Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1741,"task":"crutest"} {"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:25.722323776Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1741} {"msg":"4dc33c9a-5278-471e-8b9b-9156f53b9928 active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:25.722341736Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1741,"session_id":"3c2f5a8d-cec2-404e-8c9f-c94ae1986c50"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:25.722350625Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1741,"client":"0","":"downstairs","session_id":"3c2f5a8d-cec2-404e-8c9f-c94ae1986c50"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:25.722360205Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1741,"client":"1","":"downstairs","session_id":"3c2f5a8d-cec2-404e-8c9f-c94ae1986c50"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:25.722369595Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1741,"client":"2","":"downstairs","session_id":"3c2f5a8d-cec2-404e-8c9f-c94ae1986c50"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:25.722379235Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1741,"":"io task","client":"2","":"downstairs","session_id":"3c2f5a8d-cec2-404e-8c9f-c94ae1986c50"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:25.722398434Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1741,"":"io task","client":"1","":"downstairs","session_id":"3c2f5a8d-cec2-404e-8c9f-c94ae1986c50"} {"msg":"ds_connection connected from Ok(127.0.0.1:59806)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:25.722432393Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1741,"":"io task","client":"2","":"downstairs","session_id":"3c2f5a8d-cec2-404e-8c9f-c94ae1986c50"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:25.722442873Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1741,"":"io task","client":"0","":"downstairs","session_id":"3c2f5a8d-cec2-404e-8c9f-c94ae1986c50"} {"msg":"ds_connection connected from Ok(127.0.0.1:40150)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:25.722490851Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1741,"":"io task","client":"1","":"downstairs","session_id":"3c2f5a8d-cec2-404e-8c9f-c94ae1986c50"} {"msg":"ds_connection connected from Ok(127.0.0.1:58355)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:25.722600918Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1741,"":"io task","client":"0","":"downstairs","session_id":"3c2f5a8d-cec2-404e-8c9f-c94ae1986c50"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:25.723077664Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1741,"client":"1","":"downstairs","session_id":"3c2f5a8d-cec2-404e-8c9f-c94ae1986c50"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:25.723132693Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1741,"client":"0","":"downstairs","session_id":"3c2f5a8d-cec2-404e-8c9f-c94ae1986c50"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:25.72323786Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1741,"client":"2","":"downstairs","session_id":"3c2f5a8d-cec2-404e-8c9f-c94ae1986c50"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:25.723381836Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1741,"session_id":"3c2f5a8d-cec2-404e-8c9f-c94ae1986c50"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:25.723485423Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1741,"session_id":"3c2f5a8d-cec2-404e-8c9f-c94ae1986c50"} {"msg":"[0]R flush_numbers[0..12]: [240, 235, 249, 240, 235, 239, 248, 242, 248, 239, 246, 245]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:25.723550981Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1741,"":"downstairs","":"downstairs","session_id":"3c2f5a8d-cec2-404e-8c9f-c94ae1986c50"} {"msg":"[0]R generation[0..12]: [147, 145, 151, 147, 145, 146, 150, 148, 150, 146, 148, 148]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:25.723562261Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1741,"":"downstairs","":"downstairs","session_id":"3c2f5a8d-cec2-404e-8c9f-c94ae1986c50"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:25.72357325Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1741,"":"downstairs","":"downstairs","session_id":"3c2f5a8d-cec2-404e-8c9f-c94ae1986c50"} {"msg":"[1]R flush_numbers[0..12]: [240, 235, 249, 240, 235, 239, 248, 242, 248, 239, 246, 245]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:25.72358381Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1741,"":"downstairs","":"downstairs","session_id":"3c2f5a8d-cec2-404e-8c9f-c94ae1986c50"} {"msg":"[1]R generation[0..12]: [147, 145, 151, 147, 145, 146, 150, 148, 150, 146, 148, 148]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:25.72359417Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1741,"":"downstairs","":"downstairs","session_id":"3c2f5a8d-cec2-404e-8c9f-c94ae1986c50"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:25.723604889Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1741,"":"downstairs","":"downstairs","session_id":"3c2f5a8d-cec2-404e-8c9f-c94ae1986c50"} {"msg":"[2]R flush_numbers[0..12]: [240, 235, 249, 240, 235, 239, 248, 242, 248, 239, 246, 245]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:25.723615449Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1741,"":"downstairs","":"downstairs","session_id":"3c2f5a8d-cec2-404e-8c9f-c94ae1986c50"} {"msg":"[2]R generation[0..12]: [147, 145, 151, 147, 145, 146, 150, 148, 150, 146, 148, 148]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:25.723625909Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1741,"":"downstairs","":"downstairs","session_id":"3c2f5a8d-cec2-404e-8c9f-c94ae1986c50"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:25.723636708Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1741,"":"downstairs","":"downstairs","session_id":"3c2f5a8d-cec2-404e-8c9f-c94ae1986c50"} {"msg":"Max found gen is 152","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:25.723647608Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1741,"":"downstairs","":"downstairs","session_id":"3c2f5a8d-cec2-404e-8c9f-c94ae1986c50"} {"msg":"Generation requested: 152 >= found:152","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:25.723657258Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1741,"":"downstairs","":"downstairs","session_id":"3c2f5a8d-cec2-404e-8c9f-c94ae1986c50"} {"msg":"Next flush: 250","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:25.723666588Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1741,"":"downstairs","":"downstairs","session_id":"3c2f5a8d-cec2-404e-8c9f-c94ae1986c50"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:25.723675987Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1741,"":"downstairs","":"downstairs","session_id":"3c2f5a8d-cec2-404e-8c9f-c94ae1986c50"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:25.723685697Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1741,"session_id":"3c2f5a8d-cec2-404e-8c9f-c94ae1986c50"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:25.723694437Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1741,"client":"0","":"downstairs","session_id":"3c2f5a8d-cec2-404e-8c9f-c94ae1986c50"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:25.723704076Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1741,"client":"1","":"downstairs","session_id":"3c2f5a8d-cec2-404e-8c9f-c94ae1986c50"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:25.723713596Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1741,"client":"2","":"downstairs","session_id":"3c2f5a8d-cec2-404e-8c9f-c94ae1986c50"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:25.723723326Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1741,"session_id":"3c2f5a8d-cec2-404e-8c9f-c94ae1986c50"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:25.723732296Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1741,"session_id":"3c2f5a8d-cec2-404e-8c9f-c94ae1986c50"} {"msg":"4dc33c9a-5278-471e-8b9b-9156f53b9928 is now active with session: 3c2f5a8d-cec2-404e-8c9f-c94ae1986c50","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:25.723742155Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1741,"session_id":"3c2f5a8d-cec2-404e-8c9f-c94ae1986c50"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:25.723750745Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1741,"session_id":"3c2f5a8d-cec2-404e-8c9f-c94ae1986c50"} {"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:25.723759775Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1741} {"msg":"Activated sub_volume 4dc33c9a-5278-471e-8b9b-9156f53b9928","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:25.723767745Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1741} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:25.723775804Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1741,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:25.724126204Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1741,"task":"crutest"} 01/30 Write block 558 len 4096 data: 14 02/30 Flush 03/30 Flush 04/30 Write block 317 len 4096 data: 7 05/30 Write block 116 len 12288 data: 13 12 11 06/30 Write block 112 len 20480 data: 13 13 11 12 14 07/30 Write block 468 len 20480 data: 7 8 9 7 5 08/30 Read block 369 len 4096 09/30 Write block 180 len 16384 data: 11 10 11 10 10/30 Write block 582 len 24576 data: 14 15 14 14 15 19 11/30 Read block 69 len 8192 12/30 Read block 289 len 24576 13/30 Write block 113 len 12288 data: 14 12 13 14/30 Write block 180 len 24576 data: 12 11 12 11 13 12 15/30 Write block 4 len 4096 data: 10 16/30 Write block 140 len 28672 data: 11 14 15 15 15 14 16 17/30 Write block 76 len 8192 data: 13 14 18/30 Flush 19/30 Write block 389 len 32768 data: 9 10 10 12 13 12 15 16 20/30 Flush 21/30 Write block 170 len 24576 data: 12 10 11 13 10 10 22/30 Flush 23/30 Write block 502 len 32768 data: 16 17 18 17 14 16 10 10 24/30 Read block 450 len 36864 25/30 Flush 26/30 Write block 84 len 36864 data: 10 10 10 10 7 8 9 7 8 27/30 Read block 389 len 4096 28/30 Write block 166 len 8192 data: 12 11 29/30 Write block 179 len 24576 data: 10 13 12 13 12 14 30/30 Write block 179 len 12288 data: 11 14 13 ---------------------------------------------------------------- Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1003 Acked Write 1 Done Done Done false 1004 Acked Write 3 Done Done Sent false 1005 Acked Write 5 Done Done Sent false 1006 Acked Write 5 Done Done Sent false 1007 Acked Read 1 Done Done Sent false 1008 Acked Write 4 Done Done Sent false 1009 Acked Write 6 Done Done Sent false 1010 Acked Read 2 Done Done Sent false 1011 Acked Read 6 Done Done Sent false 1012 Acked Write 3 Done Done Sent false 1013 Acked Write 6 Done Done Sent false 1014 Acked Write 1 Done Done Sent false 1015 Acked Write 7 Done Done Sent false 1016 Acked Write 2 Done Done Sent false 1017 Acked Flush 0 Done Done Sent false 1018 Acked Write 8 Done Done Sent false 1019 Acked Flush 0 Done Done Sent false 1020 Acked Write 6 Done Done Sent false 1021 Acked Flush 0 Done Done Sent false 1022 Acked Write 8 Done Done Sent false 1023 Acked Read 9 Done Done Sent false 1024 Acked Flush 0 Done Done Sent false 1025 Acked Write 9 Done Done Sent false 1026 Acked Read 1 Done Done Sent false 1027 Acked Write 2 Done Done Sent false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 26 30 Done 25 25 1 51 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1002)) Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:47:25.731709825Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1741} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:25.73189004Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1741,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 152 152 147 252 252 240 F F F <--- 1 020-039 145 145 145 235 235 235 F F F 2 040-059 151 151 151 249 249 249 F F F 3 060-079 152 152 147 252 252 240 F F F <--- 4 080-099 145 145 145 235 235 235 T T F <--- 5 100-119 152 152 146 252 252 239 F F F <--- 6 120-139 150 150 150 248 248 248 F F F 7 140-159 152 152 148 252 252 242 F F F <--- 8 160-179 152 152 150 254 254 248 T T F <--- 9 180-199 152 152 146 252 252 239 T T F <--- 10 200-219 148 148 148 246 246 246 F F F 11 220-239 148 148 148 245 245 245 F F F 12 240-259 148 148 148 245 245 245 F F F 13 260-279 151 151 151 249 249 249 F F F 14 280-299 151 151 151 249 249 249 F F F 15 300-319 152 152 142 252 252 232 F F T <--- 16 320-339 148 148 148 241 241 241 F F F 17 340-359 140 140 140 225 225 225 F F F 18 360-379 151 151 151 249 249 249 F F F 19 380-399 152 152 151 253 253 249 F F F <--- 20 400-419 149 149 149 247 247 247 F F F 21 420-439 134 134 134 218 218 218 F F F 22 440-459 149 149 149 247 247 247 F F F 23 460-479 152 152 151 252 252 249 F F F <--- 24 480-499 151 151 151 249 249 249 F F F 25 500-519 152 152 148 255 255 241 F F F <--- 26 520-539 151 151 151 249 249 249 F F F 27 540-559 152 152 152 250 250 250 F F F 28 560-579 148 148 148 243 243 243 F F F 29 580-599 152 152 146 252 252 237 F F F <--- Max gen: 152, Max flush: 255 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 76 Verifying data now No change after verify Loop: 76 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 152 152 152 252 252 252 F F F 1 020-039 145 145 145 235 235 235 F F F 2 040-059 151 151 151 249 249 249 F F F 3 060-079 152 152 152 252 252 252 F F F 4 080-099 153 153 153 256 256 256 F F F 5 100-119 152 152 152 252 252 252 F F F 6 120-139 150 150 150 248 248 248 F F F 7 140-159 152 152 152 252 252 252 F F F 8 160-179 153 153 153 256 256 256 F F F 9 180-199 153 153 153 256 256 256 F F F 10 200-219 148 148 148 246 246 246 F F F 11 220-239 148 148 148 245 245 245 F F F 12 240-259 148 148 148 245 245 245 F F F 13 260-279 151 151 151 249 249 249 F F F 14 280-299 151 151 151 249 249 249 F F F 15 300-319 152 152 152 252 252 252 F F F 16 320-339 148 148 148 241 241 241 F F F 17 340-359 140 140 140 225 225 225 F F F 18 360-379 151 151 151 249 249 249 F F F 19 380-399 152 152 152 253 253 253 F F F 20 400-419 149 149 149 247 247 247 F F F 21 420-439 134 134 134 218 218 218 F F F 22 440-459 149 149 149 247 247 247 F F F 23 460-479 152 152 152 252 252 252 F F F 24 480-499 151 151 151 249 249 249 F F F 25 500-519 152 152 152 255 255 255 F F F 26 520-539 151 151 151 249 249 249 F F F 27 540-559 152 152 152 250 250 250 F F F 28 560-579 148 148 148 243 243 243 F F F 29 580-599 152 152 152 252 252 252 F F F Max gen: 153, Max flush: 256 Begin loop 77 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:29.977661352Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1749,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:29.985666121Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1749,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:29.985948643Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1749} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:29.985972942Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1749} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:29.985981532Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1749} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:29.985993062Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1749} {"msg":"Crucible 45f1569f-3297-4bf6-b44b-d19910222c78 has session id: 4c641dcc-8de7-4ac0-b27a-cfc9cab1e282","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:29.986349452Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1749,"session_id":"4c641dcc-8de7-4ac0-b27a-cfc9cab1e282"} {"msg":"Upstairs opts: Upstairs UUID: 45f1569f-3297-4bf6-b44b-d19910222c78, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:29.986374891Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1749,"session_id":"4c641dcc-8de7-4ac0-b27a-cfc9cab1e282"} {"msg":"Crucible stats registered with UUID: 45f1569f-3297-4bf6-b44b-d19910222c78","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:29.986384551Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1749,"session_id":"4c641dcc-8de7-4ac0-b27a-cfc9cab1e282"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:47:29.98639355Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1749,"session_id":"4c641dcc-8de7-4ac0-b27a-cfc9cab1e282"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:29.986563745Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1749,"":"io task","client":"2","":"downstairs","session_id":"4c641dcc-8de7-4ac0-b27a-cfc9cab1e282"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:29.986596324Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1749,"":"io task","client":"0","":"downstairs","session_id":"4c641dcc-8de7-4ac0-b27a-cfc9cab1e282"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:29.986607054Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1749,"":"io task","client":"1","":"downstairs","session_id":"4c641dcc-8de7-4ac0-b27a-cfc9cab1e282"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:29.986707131Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1749,"task":"crutest"} {"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:29.986721521Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1749} {"msg":"45f1569f-3297-4bf6-b44b-d19910222c78 active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:29.98675728Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1749,"session_id":"4c641dcc-8de7-4ac0-b27a-cfc9cab1e282"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:29.986766899Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1749,"client":"0","":"downstairs","session_id":"4c641dcc-8de7-4ac0-b27a-cfc9cab1e282"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:29.986776549Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1749,"client":"1","":"downstairs","session_id":"4c641dcc-8de7-4ac0-b27a-cfc9cab1e282"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:29.986786199Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1749,"client":"2","":"downstairs","session_id":"4c641dcc-8de7-4ac0-b27a-cfc9cab1e282"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:29.986795779Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1749,"":"io task","client":"2","":"downstairs","session_id":"4c641dcc-8de7-4ac0-b27a-cfc9cab1e282"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:29.986805698Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1749,"":"io task","client":"0","":"downstairs","session_id":"4c641dcc-8de7-4ac0-b27a-cfc9cab1e282"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:29.986824138Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1749,"":"io task","client":"1","":"downstairs","session_id":"4c641dcc-8de7-4ac0-b27a-cfc9cab1e282"} {"msg":"ds_connection connected from Ok(127.0.0.1:47693)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:29.986835038Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1749,"":"io task","client":"2","":"downstairs","session_id":"4c641dcc-8de7-4ac0-b27a-cfc9cab1e282"} {"msg":"ds_connection connected from Ok(127.0.0.1:37685)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:29.986854607Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1749,"":"io task","client":"0","":"downstairs","session_id":"4c641dcc-8de7-4ac0-b27a-cfc9cab1e282"} {"msg":"ds_connection connected from Ok(127.0.0.1:55531)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:29.986902686Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1749,"":"io task","client":"1","":"downstairs","session_id":"4c641dcc-8de7-4ac0-b27a-cfc9cab1e282"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:29.987509708Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1749,"client":"2","":"downstairs","session_id":"4c641dcc-8de7-4ac0-b27a-cfc9cab1e282"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:29.987543507Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1749,"client":"1","":"downstairs","session_id":"4c641dcc-8de7-4ac0-b27a-cfc9cab1e282"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:29.987624715Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1749,"client":"0","":"downstairs","session_id":"4c641dcc-8de7-4ac0-b27a-cfc9cab1e282"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:29.987883327Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1749,"session_id":"4c641dcc-8de7-4ac0-b27a-cfc9cab1e282"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:29.987907217Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1749,"session_id":"4c641dcc-8de7-4ac0-b27a-cfc9cab1e282"} {"msg":"[0]R flush_numbers[0..12]: [252, 235, 249, 252, 256, 252, 248, 252, 256, 256, 246, 245]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:29.987928116Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1749,"":"downstairs","":"downstairs","session_id":"4c641dcc-8de7-4ac0-b27a-cfc9cab1e282"} {"msg":"[0]R generation[0..12]: [152, 145, 151, 152, 153, 152, 150, 152, 153, 153, 148, 148]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:29.987942016Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1749,"":"downstairs","":"downstairs","session_id":"4c641dcc-8de7-4ac0-b27a-cfc9cab1e282"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:29.987953055Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1749,"":"downstairs","":"downstairs","session_id":"4c641dcc-8de7-4ac0-b27a-cfc9cab1e282"} {"msg":"[1]R flush_numbers[0..12]: [252, 235, 249, 252, 256, 252, 248, 252, 256, 256, 246, 245]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:29.987963675Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1749,"":"downstairs","":"downstairs","session_id":"4c641dcc-8de7-4ac0-b27a-cfc9cab1e282"} {"msg":"[1]R generation[0..12]: [152, 145, 151, 152, 153, 152, 150, 152, 153, 153, 148, 148]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:29.987974125Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1749,"":"downstairs","":"downstairs","session_id":"4c641dcc-8de7-4ac0-b27a-cfc9cab1e282"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:29.987984914Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1749,"":"downstairs","":"downstairs","session_id":"4c641dcc-8de7-4ac0-b27a-cfc9cab1e282"} {"msg":"[2]R flush_numbers[0..12]: [252, 235, 249, 252, 256, 252, 248, 252, 256, 256, 246, 245]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:29.987995454Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1749,"":"downstairs","":"downstairs","session_id":"4c641dcc-8de7-4ac0-b27a-cfc9cab1e282"} {"msg":"[2]R generation[0..12]: [152, 145, 151, 152, 153, 152, 150, 152, 153, 153, 148, 148]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:29.988018083Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1749,"":"downstairs","":"downstairs","session_id":"4c641dcc-8de7-4ac0-b27a-cfc9cab1e282"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:29.988029023Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1749,"":"downstairs","":"downstairs","session_id":"4c641dcc-8de7-4ac0-b27a-cfc9cab1e282"} {"msg":"Max found gen is 154","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:29.988038563Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1749,"":"downstairs","":"downstairs","session_id":"4c641dcc-8de7-4ac0-b27a-cfc9cab1e282"} {"msg":"Generation requested: 154 >= found:154","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:29.988048443Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1749,"":"downstairs","":"downstairs","session_id":"4c641dcc-8de7-4ac0-b27a-cfc9cab1e282"} {"msg":"Next flush: 257","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:29.988057892Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1749,"":"downstairs","":"downstairs","session_id":"4c641dcc-8de7-4ac0-b27a-cfc9cab1e282"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:29.988067272Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1749,"":"downstairs","":"downstairs","session_id":"4c641dcc-8de7-4ac0-b27a-cfc9cab1e282"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:29.988077212Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1749,"session_id":"4c641dcc-8de7-4ac0-b27a-cfc9cab1e282"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:29.988085971Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1749,"client":"0","":"downstairs","session_id":"4c641dcc-8de7-4ac0-b27a-cfc9cab1e282"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:29.988096971Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1749,"client":"1","":"downstairs","session_id":"4c641dcc-8de7-4ac0-b27a-cfc9cab1e282"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:29.988107051Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1749,"client":"2","":"downstairs","session_id":"4c641dcc-8de7-4ac0-b27a-cfc9cab1e282"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:29.988117091Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1749,"session_id":"4c641dcc-8de7-4ac0-b27a-cfc9cab1e282"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:29.98812634Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1749,"session_id":"4c641dcc-8de7-4ac0-b27a-cfc9cab1e282"} {"msg":"45f1569f-3297-4bf6-b44b-d19910222c78 is now active with session: 4c641dcc-8de7-4ac0-b27a-cfc9cab1e282","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:29.98813645Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1749,"session_id":"4c641dcc-8de7-4ac0-b27a-cfc9cab1e282"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:29.98814525Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1749,"session_id":"4c641dcc-8de7-4ac0-b27a-cfc9cab1e282"} {"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:29.988154479Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1749} {"msg":"Activated sub_volume 45f1569f-3297-4bf6-b44b-d19910222c78","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:29.988162609Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1749} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:29.988170779Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1749,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:29.9884917Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1749,"task":"crutest"} 01/30 Flush 02/30 Write block 286 len 36864 data: 11 10 10 11 11 11 12 15 14 03/30 Read block 88 len 36864 04/30 Read block 72 len 12288 05/30 Write block 74 len 40960 data: 15 14 14 15 16 14 10 8 9 10 06/30 Write block 473 len 28672 data: 5 9 13 18 18 18 17 07/30 Write block 66 len 20480 data: 6 7 5 7 11 08/30 Write block 414 len 28672 data: 12 9 14 14 13 13 13 09/30 Read block 490 len 28672 10/30 Write block 572 len 4096 data: 10 11/30 Read block 468 len 32768 12/30 Write block 333 len 16384 data: 12 12 13 14 13/30 Flush 14/30 Write block 503 len 8192 data: 18 19 15/30 Read block 257 len 16384 16/30 Read block 8 len 8192 17/30 Read block 478 len 36864 18/30 Read block 473 len 16384 19/30 Write block 555 len 4096 data: 9 20/30 Flush 21/30 Write block 388 len 16384 data: 12 10 11 11 22/30 Read block 483 len 20480 23/30 Read block 271 len 28672 24/30 Write block 415 len 16384 data: 10 15 15 14 25/30 Read block 107 len 16384 26/30 Read block 96 len 12288 27/30 Read block 545 len 32768 28/30 Write block 172 len 36864 data: 12 14 11 11 9 10 10 12 15 29/30 Read block 535 len 32768 30/30 Read block 590 len 8192 ---------------------------------------------------------------- Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 9 Done Sent Done false 1002 Acked Read 9 Done Sent Done false 1003 Acked Read 3 Done Sent Done false 1004 Acked Write 10 Done Sent Done false 1005 Acked Write 7 Done Sent Done false 1006 Acked Write 5 Done Sent Done false 1007 Acked Write 7 Done Sent Done false 1008 Acked Read 7 Done Sent Done false 1009 Acked Write 1 Done Sent Done false 1010 Acked Read 8 Done Sent Done false 1011 Acked Write 4 Done Sent Done false 1012 Acked Flush 0 Done Sent Done false 1013 Acked Write 2 Done Sent Done false 1014 Acked Read 4 Done Sent Done false 1015 Acked Read 2 Done Sent Done false 1016 Acked Read 9 Done Sent Done false 1017 Acked Read 4 Done Sent Done false 1018 Acked Write 1 Done Sent Done false 1019 Acked Flush 0 Done Sent Done false 1020 Acked Write 4 Done Sent Done false 1021 Acked Read 5 Done Sent Done false 1022 Acked Read 7 Done Sent Done false 1023 Acked Write 4 Done Sent Done false 1024 Acked Read 4 Done Sent Done false 1025 Acked Read 3 Done Sent Done false 1026 Acked Read 8 Done Sent Done false 1027 Acked Write 9 Done Sent Done false 1028 Acked Read 8 Done Sent Done false 1029 Acked Read 2 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 29 1 30 Done 29 0 28 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1000)) Some(JobId(1019)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:47:29.997540149Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1749} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:29.997727093Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1749,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 152 152 152 252 252 252 F F F 1 020-039 145 145 145 235 235 235 F F F 2 040-059 151 151 151 249 249 249 F F F 3 060-079 154 152 154 258 252 258 F F F <--- 4 080-099 154 153 154 258 256 258 F F F <--- 5 100-119 152 152 152 252 252 252 F F F 6 120-139 150 150 150 248 248 248 F F F 7 140-159 152 152 152 252 252 252 F F F 8 160-179 153 153 153 256 256 256 T F T <--- 9 180-199 153 153 153 256 256 256 T F T <--- 10 200-219 148 148 148 246 246 246 F F F 11 220-239 148 148 148 245 245 245 F F F 12 240-259 148 148 148 245 245 245 F F F 13 260-279 151 151 151 249 249 249 F F F 14 280-299 154 151 154 258 249 258 F F F <--- 15 300-319 152 152 152 252 252 252 F F F 16 320-339 154 148 154 258 241 258 F F F <--- 17 340-359 140 140 140 225 225 225 F F F 18 360-379 151 151 151 249 249 249 F F F 19 380-399 152 152 152 253 253 253 T F T <--- 20 400-419 154 149 154 258 247 258 T F T <--- 21 420-439 154 134 154 258 218 258 F F F <--- 22 440-459 149 149 149 247 247 247 F F F 23 460-479 154 152 154 258 252 258 F F F <--- 24 480-499 151 151 151 249 249 249 F F F 25 500-519 154 152 154 259 255 259 F F F <--- 26 520-539 151 151 151 249 249 249 F F F 27 540-559 154 152 154 259 250 259 F F F <--- 28 560-579 154 148 154 258 243 258 F F F <--- 29 580-599 152 152 152 252 252 252 F F F Max gen: 154, Max flush: 259 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 77 Verifying data now No change after verify Loop: 77 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 152 152 152 252 252 252 F F F 1 020-039 145 145 145 235 235 235 F F F 2 040-059 151 151 151 249 249 249 F F F 3 060-079 154 154 154 258 258 258 F F F 4 080-099 154 154 154 258 258 258 F F F 5 100-119 152 152 152 252 252 252 F F F 6 120-139 150 150 150 248 248 248 F F F 7 140-159 152 152 152 252 252 252 F F F 8 160-179 155 155 155 260 260 260 F F F 9 180-199 155 155 155 260 260 260 F F F 10 200-219 148 148 148 246 246 246 F F F 11 220-239 148 148 148 245 245 245 F F F 12 240-259 148 148 148 245 245 245 F F F 13 260-279 151 151 151 249 249 249 F F F 14 280-299 154 154 154 258 258 258 F F F 15 300-319 152 152 152 252 252 252 F F F 16 320-339 154 154 154 258 258 258 F F F 17 340-359 140 140 140 225 225 225 F F F 18 360-379 151 151 151 249 249 249 F F F 19 380-399 155 155 155 260 260 260 F F F 20 400-419 155 155 155 260 260 260 F F F 21 420-439 154 154 154 258 258 258 F F F 22 440-459 149 149 149 247 247 247 F F F 23 460-479 154 154 154 258 258 258 F F F 24 480-499 151 151 151 249 249 249 F F F 25 500-519 154 154 154 259 259 259 F F F 26 520-539 151 151 151 249 249 249 F F F 27 540-559 154 154 154 259 259 259 F F F 28 560-579 154 154 154 258 258 258 F F F 29 580-599 152 152 152 252 252 252 F F F Max gen: 155, Max flush: 260 Begin loop 78 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:34.244173786Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1757,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:34.252139016Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1757,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:34.252437968Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1757} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:34.252475467Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1757} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:34.252484646Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1757} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:34.252496166Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1757} {"msg":"Crucible 17d087cd-6afe-4ccd-8828-d5dd3c7d8949 has session id: 0b51c8a1-c668-4b90-a801-dfb82ccd2cfd","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:34.252826927Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1757,"session_id":"0b51c8a1-c668-4b90-a801-dfb82ccd2cfd"} {"msg":"Upstairs opts: Upstairs UUID: 17d087cd-6afe-4ccd-8828-d5dd3c7d8949, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:34.252847796Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1757,"session_id":"0b51c8a1-c668-4b90-a801-dfb82ccd2cfd"} {"msg":"Crucible stats registered with UUID: 17d087cd-6afe-4ccd-8828-d5dd3c7d8949","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:34.252857886Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1757,"session_id":"0b51c8a1-c668-4b90-a801-dfb82ccd2cfd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:47:34.252871495Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1757,"session_id":"0b51c8a1-c668-4b90-a801-dfb82ccd2cfd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:34.253028111Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1757,"":"io task","client":"1","":"downstairs","session_id":"0b51c8a1-c668-4b90-a801-dfb82ccd2cfd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:34.25304598Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1757,"":"io task","client":"0","":"downstairs","session_id":"0b51c8a1-c668-4b90-a801-dfb82ccd2cfd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:34.25305615Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1757,"":"io task","client":"2","":"downstairs","session_id":"0b51c8a1-c668-4b90-a801-dfb82ccd2cfd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:34.253199316Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1757,"task":"crutest"} {"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:34.253213445Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1757} {"msg":"17d087cd-6afe-4ccd-8828-d5dd3c7d8949 active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:34.253248044Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1757,"session_id":"0b51c8a1-c668-4b90-a801-dfb82ccd2cfd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:34.253257354Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1757,"client":"0","":"downstairs","session_id":"0b51c8a1-c668-4b90-a801-dfb82ccd2cfd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:34.253267084Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1757,"client":"1","":"downstairs","session_id":"0b51c8a1-c668-4b90-a801-dfb82ccd2cfd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:34.253276654Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1757,"client":"2","":"downstairs","session_id":"0b51c8a1-c668-4b90-a801-dfb82ccd2cfd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:34.253285973Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1757,"":"io task","client":"2","":"downstairs","session_id":"0b51c8a1-c668-4b90-a801-dfb82ccd2cfd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:34.253296033Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1757,"":"io task","client":"0","":"downstairs","session_id":"0b51c8a1-c668-4b90-a801-dfb82ccd2cfd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:34.253307573Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1757,"":"io task","client":"1","":"downstairs","session_id":"0b51c8a1-c668-4b90-a801-dfb82ccd2cfd"} {"msg":"ds_connection connected from Ok(127.0.0.1:47452)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:34.253369821Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1757,"":"io task","client":"1","":"downstairs","session_id":"0b51c8a1-c668-4b90-a801-dfb82ccd2cfd"} {"msg":"ds_connection connected from Ok(127.0.0.1:45338)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:34.25339564Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1757,"":"io task","client":"2","":"downstairs","session_id":"0b51c8a1-c668-4b90-a801-dfb82ccd2cfd"} {"msg":"ds_connection connected from Ok(127.0.0.1:40607)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:34.25340709Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1757,"":"io task","client":"0","":"downstairs","session_id":"0b51c8a1-c668-4b90-a801-dfb82ccd2cfd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:34.254058391Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1757,"client":"2","":"downstairs","session_id":"0b51c8a1-c668-4b90-a801-dfb82ccd2cfd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:34.25409354Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1757,"client":"0","":"downstairs","session_id":"0b51c8a1-c668-4b90-a801-dfb82ccd2cfd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:34.25410484Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1757,"client":"1","":"downstairs","session_id":"0b51c8a1-c668-4b90-a801-dfb82ccd2cfd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:34.2544338Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1757,"session_id":"0b51c8a1-c668-4b90-a801-dfb82ccd2cfd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:34.25444435Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1757,"session_id":"0b51c8a1-c668-4b90-a801-dfb82ccd2cfd"} {"msg":"[0]R flush_numbers[0..12]: [252, 235, 249, 258, 258, 252, 248, 252, 260, 260, 246, 245]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:34.254483999Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1757,"":"downstairs","":"downstairs","session_id":"0b51c8a1-c668-4b90-a801-dfb82ccd2cfd"} {"msg":"[0]R generation[0..12]: [152, 145, 151, 154, 154, 152, 150, 152, 155, 155, 148, 148]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:34.254495298Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1757,"":"downstairs","":"downstairs","session_id":"0b51c8a1-c668-4b90-a801-dfb82ccd2cfd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:34.254506298Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1757,"":"downstairs","":"downstairs","session_id":"0b51c8a1-c668-4b90-a801-dfb82ccd2cfd"} {"msg":"[1]R flush_numbers[0..12]: [252, 235, 249, 258, 258, 252, 248, 252, 260, 260, 246, 245]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:34.254516938Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1757,"":"downstairs","":"downstairs","session_id":"0b51c8a1-c668-4b90-a801-dfb82ccd2cfd"} {"msg":"[1]R generation[0..12]: [152, 145, 151, 154, 154, 152, 150, 152, 155, 155, 148, 148]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:34.254527408Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1757,"":"downstairs","":"downstairs","session_id":"0b51c8a1-c668-4b90-a801-dfb82ccd2cfd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:34.254540727Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1757,"":"downstairs","":"downstairs","session_id":"0b51c8a1-c668-4b90-a801-dfb82ccd2cfd"} {"msg":"[2]R flush_numbers[0..12]: [252, 235, 249, 258, 258, 252, 248, 252, 260, 260, 246, 245]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:34.254551417Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1757,"":"downstairs","":"downstairs","session_id":"0b51c8a1-c668-4b90-a801-dfb82ccd2cfd"} {"msg":"[2]R generation[0..12]: [152, 145, 151, 154, 154, 152, 150, 152, 155, 155, 148, 148]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:34.254561947Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1757,"":"downstairs","":"downstairs","session_id":"0b51c8a1-c668-4b90-a801-dfb82ccd2cfd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:34.254572766Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1757,"":"downstairs","":"downstairs","session_id":"0b51c8a1-c668-4b90-a801-dfb82ccd2cfd"} {"msg":"Max found gen is 156","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:34.254582076Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1757,"":"downstairs","":"downstairs","session_id":"0b51c8a1-c668-4b90-a801-dfb82ccd2cfd"} {"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:34.254591966Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1757,"":"downstairs","":"downstairs","session_id":"0b51c8a1-c668-4b90-a801-dfb82ccd2cfd"} {"msg":"Next flush: 261","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:34.254601285Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1757,"":"downstairs","":"downstairs","session_id":"0b51c8a1-c668-4b90-a801-dfb82ccd2cfd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:34.254610575Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1757,"":"downstairs","":"downstairs","session_id":"0b51c8a1-c668-4b90-a801-dfb82ccd2cfd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:34.254620345Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1757,"session_id":"0b51c8a1-c668-4b90-a801-dfb82ccd2cfd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:34.254629115Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1757,"client":"0","":"downstairs","session_id":"0b51c8a1-c668-4b90-a801-dfb82ccd2cfd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:34.254638834Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1757,"client":"1","":"downstairs","session_id":"0b51c8a1-c668-4b90-a801-dfb82ccd2cfd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:34.254648364Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1757,"client":"2","":"downstairs","session_id":"0b51c8a1-c668-4b90-a801-dfb82ccd2cfd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:34.254658064Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1757,"session_id":"0b51c8a1-c668-4b90-a801-dfb82ccd2cfd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:34.254667223Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1757,"session_id":"0b51c8a1-c668-4b90-a801-dfb82ccd2cfd"} {"msg":"17d087cd-6afe-4ccd-8828-d5dd3c7d8949 is now active with session: 0b51c8a1-c668-4b90-a801-dfb82ccd2cfd","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:34.254677093Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1757,"session_id":"0b51c8a1-c668-4b90-a801-dfb82ccd2cfd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:34.254685783Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1757,"session_id":"0b51c8a1-c668-4b90-a801-dfb82ccd2cfd"} {"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:34.254696163Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1757} {"msg":"Activated sub_volume 17d087cd-6afe-4ccd-8828-d5dd3c7d8949","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:34.254704192Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1757} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:34.254712372Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1757,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:34.255052022Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1757,"task":"crutest"} 01/30 Flush 02/30 Write block 381 len 28672 data: 19 22 16 14 13 11 12 03/30 Read block 317 len 36864 04/30 Write block 1 len 12288 data: 6 9 6 05/30 Read block 573 len 40960 06/30 Read block 369 len 32768 07/30 Read block 68 len 36864 08/30 Read block 52 len 32768 09/30 Write block 172 len 36864 data: 13 15 12 12 10 11 11 13 16 10/30 Read block 326 len 36864 11/30 Read block 549 len 8192 12/30 Read block 226 len 4096 13/30 Read block 446 len 32768 14/30 Read block 190 len 32768 15/30 Write block 241 len 36864 data: 13 12 12 12 12 14 16 16 13 16/30 Read block 8 len 40960 17/30 Write block 429 len 4096 data: 12 18/30 Write block 372 len 4096 data: 8 19/30 Write block 564 len 12288 data: 12 11 10 20/30 Read block 206 len 36864 21/30 Write block 261 len 12288 data: 11 12 10 22/30 Write block 484 len 24576 data: 11 13 13 15 17 17 23/30 Write block 105 len 16384 data: 4 4 5 4 24/30 Write block 185 len 40960 data: 13 14 12 10 11 10 10 10 10 12 25/30 Write block 313 len 20480 data: 13 13 13 9 8 26/30 Write block 522 len 24576 data: 15 14 12 9 7 7 27/30 Write block 323 len 40960 data: 13 18 18 19 20 16 15 13 14 11 28/30 Read block 181 len 36864 29/30 Write block 425 len 36864 data: 11 11 11 10 13 10 9 8 6 30/30 Write block 171 len 40960 data: 11 14 16 13 13 11 12 12 14 17 ---------------------------------------------------------------- Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 7 Done Done Done false 1002 Acked Read 9 Done Done Done false 1003 Acked Write 3 Done Done Done false 1004 Acked Read 10 Done Done Sent false 1005 Acked Read 8 Done Done Sent false 1006 Acked Read 9 Done Done Sent false 1007 Acked Read 8 Done Done Sent false 1008 Acked Write 9 Done Done Sent false 1009 Acked Read 9 Done Done Sent false 1010 Acked Read 2 Done Done Sent false 1011 Acked Read 1 Done Done Sent false 1012 Acked Read 8 Done Done Sent false 1013 Acked Read 8 Done Done Sent false 1014 Acked Write 9 Done Done Sent false 1015 Acked Read 10 Done Done Sent false 1016 Acked Write 1 Done Done Sent false 1017 Acked Write 1 Done Done Sent false 1018 Acked Write 3 Done Done Sent false 1019 Acked Read 9 Done Done Sent false 1020 Acked Write 3 Done Done Sent false 1021 Acked Write 6 Done Done Sent false 1022 Acked Write 4 Done Done Sent false 1023 Acked Write 10 Done Done Sent false 1024 Acked Write 5 Done Done Sent false 1025 Acked Write 6 Done Done Sent false 1026 Acked Write 10 Done Done Sent false 1027 Acked Read 9 Done Done Sent false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 26 30 Done 27 27 3 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1000)) Some(JobId(1000)) Some(JobId(1000)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:47:34.263181258Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1757} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:34.263400552Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1757,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 152 152 152 252 252 252 T T T 1 020-039 145 145 145 235 235 235 F F F 2 040-059 151 151 151 249 249 249 F F F 3 060-079 154 154 154 258 258 258 F F F 4 080-099 154 154 154 258 258 258 F F F 5 100-119 152 152 152 252 252 252 T T F <--- 6 120-139 150 150 150 248 248 248 F F F 7 140-159 152 152 152 252 252 252 F F F 8 160-179 155 155 155 260 260 260 T T F <--- 9 180-199 155 155 155 260 260 260 T T F <--- 10 200-219 148 148 148 246 246 246 F F F 11 220-239 148 148 148 245 245 245 F F F 12 240-259 148 148 148 245 245 245 T T F <--- 13 260-279 151 151 151 249 249 249 T T F <--- 14 280-299 154 154 154 258 258 258 F F F 15 300-319 152 152 152 252 252 252 T T F <--- 16 320-339 154 154 154 258 258 258 T T F <--- 17 340-359 140 140 140 225 225 225 F F F 18 360-379 151 151 151 249 249 249 T T F <--- 19 380-399 155 155 155 260 260 260 T T T 20 400-419 155 155 155 260 260 260 F F F 21 420-439 154 154 154 258 258 258 T T F <--- 22 440-459 149 149 149 247 247 247 F F F 23 460-479 154 154 154 258 258 258 F F F 24 480-499 151 151 151 249 249 249 T T F <--- 25 500-519 154 154 154 259 259 259 F F F 26 520-539 151 151 151 249 249 249 T T F <--- 27 540-559 154 154 154 259 259 259 F F F 28 560-579 154 154 154 258 258 258 T T F <--- 29 580-599 152 152 152 252 252 252 F F F Max gen: 155, Max flush: 260 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 78 Verifying data now No change after verify Loop: 78 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 261 261 261 F F F 1 020-039 145 145 145 235 235 235 F F F 2 040-059 151 151 151 249 249 249 F F F 3 060-079 154 154 154 258 258 258 F F F 4 080-099 154 154 154 258 258 258 F F F 5 100-119 156 156 156 261 261 261 F F F 6 120-139 150 150 150 248 248 248 F F F 7 140-159 152 152 152 252 252 252 F F F 8 160-179 156 156 156 261 261 261 F F F 9 180-199 156 156 156 261 261 261 F F F 10 200-219 148 148 148 246 246 246 F F F 11 220-239 148 148 148 245 245 245 F F F 12 240-259 156 156 156 261 261 261 F F F 13 260-279 156 156 156 261 261 261 F F F 14 280-299 154 154 154 258 258 258 F F F 15 300-319 156 156 156 261 261 261 F F F 16 320-339 156 156 156 261 261 261 F F F 17 340-359 140 140 140 225 225 225 F F F 18 360-379 156 156 156 261 261 261 F F F 19 380-399 156 156 156 261 261 261 F F F 20 400-419 155 155 155 260 260 260 F F F 21 420-439 156 156 156 261 261 261 F F F 22 440-459 149 149 149 247 247 247 F F F 23 460-479 154 154 154 258 258 258 F F F 24 480-499 156 156 156 261 261 261 F F F 25 500-519 154 154 154 259 259 259 F F F 26 520-539 156 156 156 261 261 261 F F F 27 540-559 154 154 154 259 259 259 F F F 28 560-579 156 156 156 261 261 261 F F F 29 580-599 152 152 152 252 252 252 F F F Max gen: 156, Max flush: 261 Begin loop 79 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:38.51935157Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1765,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:47:38.526405017Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1765,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:38.526456185Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1765,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:38.53322205Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1765,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:38.533488942Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1765} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:38.533516202Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1765} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:38.533524821Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1765} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:38.533536321Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1765} {"msg":"Crucible c70a4cc0-758f-43c7-9e9d-61460eb3895f has session id: b81454ac-643b-4df0-9e88-2b97f021c7aa","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:38.533856152Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1765,"session_id":"b81454ac-643b-4df0-9e88-2b97f021c7aa"} {"msg":"Upstairs opts: Upstairs UUID: c70a4cc0-758f-43c7-9e9d-61460eb3895f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:38.533881311Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1765,"session_id":"b81454ac-643b-4df0-9e88-2b97f021c7aa"} {"msg":"Crucible stats registered with UUID: c70a4cc0-758f-43c7-9e9d-61460eb3895f","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:38.533893671Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1765,"session_id":"b81454ac-643b-4df0-9e88-2b97f021c7aa"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:47:38.533902701Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1765,"session_id":"b81454ac-643b-4df0-9e88-2b97f021c7aa"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:38.534044816Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1765,"":"io task","client":"0","":"downstairs","session_id":"b81454ac-643b-4df0-9e88-2b97f021c7aa"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:38.534062676Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1765,"":"io task","client":"1","":"downstairs","session_id":"b81454ac-643b-4df0-9e88-2b97f021c7aa"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:38.534072726Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1765,"":"io task","client":"2","":"downstairs","session_id":"b81454ac-643b-4df0-9e88-2b97f021c7aa"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:38.534201882Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1765,"task":"crutest"} {"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:38.534218381Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1765} {"msg":"c70a4cc0-758f-43c7-9e9d-61460eb3895f active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:38.534235581Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1765,"session_id":"b81454ac-643b-4df0-9e88-2b97f021c7aa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:38.534244391Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1765,"client":"0","":"downstairs","session_id":"b81454ac-643b-4df0-9e88-2b97f021c7aa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:38.53425408Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1765,"client":"1","":"downstairs","session_id":"b81454ac-643b-4df0-9e88-2b97f021c7aa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:38.53426371Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1765,"client":"2","":"downstairs","session_id":"b81454ac-643b-4df0-9e88-2b97f021c7aa"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:38.53427335Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1765,"":"io task","client":"2","":"downstairs","session_id":"b81454ac-643b-4df0-9e88-2b97f021c7aa"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:38.53428328Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1765,"":"io task","client":"0","":"downstairs","session_id":"b81454ac-643b-4df0-9e88-2b97f021c7aa"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:38.534293309Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1765,"":"io task","client":"1","":"downstairs","session_id":"b81454ac-643b-4df0-9e88-2b97f021c7aa"} {"msg":"ds_connection connected from Ok(127.0.0.1:49954)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:38.534388077Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1765,"":"io task","client":"2","":"downstairs","session_id":"b81454ac-643b-4df0-9e88-2b97f021c7aa"} {"msg":"ds_connection connected from Ok(127.0.0.1:32811)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:38.534496053Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1765,"":"io task","client":"1","":"downstairs","session_id":"b81454ac-643b-4df0-9e88-2b97f021c7aa"} {"msg":"ds_connection connected from Ok(127.0.0.1:35244)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:38.534519933Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1765,"":"io task","client":"0","":"downstairs","session_id":"b81454ac-643b-4df0-9e88-2b97f021c7aa"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:38.535075237Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1765,"client":"2","":"downstairs","session_id":"b81454ac-643b-4df0-9e88-2b97f021c7aa"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:38.535092186Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1765,"client":"1","":"downstairs","session_id":"b81454ac-643b-4df0-9e88-2b97f021c7aa"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:38.535107356Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1765,"client":"0","":"downstairs","session_id":"b81454ac-643b-4df0-9e88-2b97f021c7aa"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:38.535394577Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1765,"session_id":"b81454ac-643b-4df0-9e88-2b97f021c7aa"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:38.535438196Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1765,"session_id":"b81454ac-643b-4df0-9e88-2b97f021c7aa"} {"msg":"[0]R flush_numbers[0..12]: [261, 235, 249, 258, 258, 261, 248, 252, 261, 261, 246, 245]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:38.535492705Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1765,"":"downstairs","":"downstairs","session_id":"b81454ac-643b-4df0-9e88-2b97f021c7aa"} {"msg":"[0]R generation[0..12]: [156, 145, 151, 154, 154, 156, 150, 152, 156, 156, 148, 148]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:38.535503914Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1765,"":"downstairs","":"downstairs","session_id":"b81454ac-643b-4df0-9e88-2b97f021c7aa"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:38.535514804Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1765,"":"downstairs","":"downstairs","session_id":"b81454ac-643b-4df0-9e88-2b97f021c7aa"} {"msg":"[1]R flush_numbers[0..12]: [261, 235, 249, 258, 258, 261, 248, 252, 261, 261, 246, 245]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:38.535525504Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1765,"":"downstairs","":"downstairs","session_id":"b81454ac-643b-4df0-9e88-2b97f021c7aa"} {"msg":"[1]R generation[0..12]: [156, 145, 151, 154, 154, 156, 150, 152, 156, 156, 148, 148]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:38.535535973Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1765,"":"downstairs","":"downstairs","session_id":"b81454ac-643b-4df0-9e88-2b97f021c7aa"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:38.535546703Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1765,"":"downstairs","":"downstairs","session_id":"b81454ac-643b-4df0-9e88-2b97f021c7aa"} {"msg":"[2]R flush_numbers[0..12]: [261, 235, 249, 258, 258, 261, 248, 252, 261, 261, 246, 245]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:38.535557243Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1765,"":"downstairs","":"downstairs","session_id":"b81454ac-643b-4df0-9e88-2b97f021c7aa"} {"msg":"[2]R generation[0..12]: [156, 145, 151, 154, 154, 156, 150, 152, 156, 156, 148, 148]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:38.535567722Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1765,"":"downstairs","":"downstairs","session_id":"b81454ac-643b-4df0-9e88-2b97f021c7aa"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:38.535581522Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1765,"":"downstairs","":"downstairs","session_id":"b81454ac-643b-4df0-9e88-2b97f021c7aa"} {"msg":"Max found gen is 157","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:38.535590862Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1765,"":"downstairs","":"downstairs","session_id":"b81454ac-643b-4df0-9e88-2b97f021c7aa"} {"msg":"Generation requested: 158 >= found:157","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:38.535600662Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1765,"":"downstairs","":"downstairs","session_id":"b81454ac-643b-4df0-9e88-2b97f021c7aa"} {"msg":"Next flush: 262","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:38.535609871Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1765,"":"downstairs","":"downstairs","session_id":"b81454ac-643b-4df0-9e88-2b97f021c7aa"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:38.535619261Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1765,"":"downstairs","":"downstairs","session_id":"b81454ac-643b-4df0-9e88-2b97f021c7aa"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:38.535629141Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1765,"session_id":"b81454ac-643b-4df0-9e88-2b97f021c7aa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:38.53563799Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1765,"client":"0","":"downstairs","session_id":"b81454ac-643b-4df0-9e88-2b97f021c7aa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:38.53564776Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1765,"client":"1","":"downstairs","session_id":"b81454ac-643b-4df0-9e88-2b97f021c7aa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:38.53565738Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1765,"client":"2","":"downstairs","session_id":"b81454ac-643b-4df0-9e88-2b97f021c7aa"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:38.535667Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1765,"session_id":"b81454ac-643b-4df0-9e88-2b97f021c7aa"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:38.535676439Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1765,"session_id":"b81454ac-643b-4df0-9e88-2b97f021c7aa"} {"msg":"c70a4cc0-758f-43c7-9e9d-61460eb3895f is now active with session: b81454ac-643b-4df0-9e88-2b97f021c7aa","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:38.535686499Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1765,"session_id":"b81454ac-643b-4df0-9e88-2b97f021c7aa"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:38.535695189Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1765,"session_id":"b81454ac-643b-4df0-9e88-2b97f021c7aa"} {"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:38.535704319Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1765} {"msg":"Activated sub_volume c70a4cc0-758f-43c7-9e9d-61460eb3895f","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:38.535712408Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1765} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:38.535720508Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1765,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:38.536058948Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1765,"task":"crutest"} 01/30 Write block 310 len 32768 data: 12 12 12 14 14 14 10 9 02/30 Read block 467 len 40960 03/30 Write block 282 len 32768 data: 8 9 10 13 12 11 11 12 04/30 Read block 300 len 24576 05/30 Write block 37 len 20480 data: 10 8 6 6 7 06/30 Write block 355 len 32768 data: 14 12 12 11 10 12 14 17 07/30 Write block 88 len 12288 data: 8 9 10 08/30 Read block 62 len 32768 09/30 Write block 555 len 24576 data: 10 8 11 15 13 13 10/30 Write block 97 len 16384 data: 14 12 11 11 11/30 Write block 77 len 32768 data: 16 17 15 11 9 10 11 11 12/30 Read block 166 len 36864 13/30 Write block 394 len 4096 data: 13 14/30 Write block 190 len 28672 data: 11 11 11 11 13 11 12 15/30 Write block 319 len 16384 data: 7 6 10 10 16/30 Read block 216 len 12288 17/30 Write block 540 len 32768 data: 19 14 14 14 14 13 11 10 18/30 Read block 381 len 16384 19/30 Read block 331 len 16384 20/30 Read block 584 len 8192 21/30 Write block 216 len 16384 data: 12 12 11 10 22/30 Write block 486 len 16384 data: 14 16 18 18 23/30 Read block 207 len 16384 24/30 Read block 30 len 16384 25/30 Write block 501 len 32768 data: 17 17 19 20 18 15 17 11 26/30 Read block 188 len 16384 27/30 Write block 542 len 12288 data: 15 15 15 28/30 Read block 225 len 40960 29/30 Read block 390 len 40960 30/30 Read block 523 len 28672 ---------------------------------------------------------------- Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Sent Done Done false 1001 Acked Read 10 Sent Done Done false 1002 Acked Write 8 Sent Done Done false 1003 Acked Read 6 Sent Done Done false 1004 Acked Write 5 Sent Done Done false 1005 Acked Write 8 Sent Done Done false 1006 Acked Write 3 Sent Done Done false 1007 Acked Read 8 Sent Done Done false 1008 Acked Write 6 Sent Done Done false 1009 Acked Write 4 Sent Done Done false 1010 Acked Write 8 Sent Done Done false 1011 Acked Read 9 Sent Done Done false 1012 Acked Write 1 Sent Done Done false 1013 Acked Write 7 Sent Done Done false 1014 Acked Write 4 Sent Done Done false 1015 Acked Read 3 Sent Done Done false 1016 Acked Write 8 Sent Done Done false 1017 Acked Read 4 Sent Done Done false 1018 Acked Read 4 Sent Done Done false 1019 Acked Read 2 Sent Done Done false 1020 Acked Write 4 Sent Done Done false 1021 Acked Write 4 Sent Done Done false 1022 Acked Read 4 Sent Done Done false 1023 Acked Read 4 Sent Done Done false 1024 Acked Write 8 Sent Done Done false 1025 Acked Read 4 Sent Done Done false 1026 Acked Write 3 Sent Done Done false 1027 Acked Read 10 Sent Done Done false 1028 Acked Read 10 Sent Done Done false 1029 Acked Read 7 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 1 0 31 Done 0 29 30 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:47:38.544748018Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1765} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:38.544938352Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1765,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 261 261 261 F F F 1 020-039 145 145 145 235 235 235 F T T <--- 2 040-059 151 151 151 249 249 249 F T T <--- 3 060-079 154 154 154 258 258 258 F T T <--- 4 080-099 154 154 154 258 258 258 F T T <--- 5 100-119 156 156 156 261 261 261 F T T <--- 6 120-139 150 150 150 248 248 248 F F F 7 140-159 152 152 152 252 252 252 F F F 8 160-179 156 156 156 261 261 261 F F F 9 180-199 156 156 156 261 261 261 F T T <--- 10 200-219 148 148 148 246 246 246 F T T <--- 11 220-239 148 148 148 245 245 245 F F F 12 240-259 156 156 156 261 261 261 F F F 13 260-279 156 156 156 261 261 261 F F F 14 280-299 154 154 154 258 258 258 F T T <--- 15 300-319 156 156 156 261 261 261 F T T <--- 16 320-339 156 156 156 261 261 261 F T T <--- 17 340-359 140 140 140 225 225 225 F T T <--- 18 360-379 156 156 156 261 261 261 F T T <--- 19 380-399 156 156 156 261 261 261 F T T <--- 20 400-419 155 155 155 260 260 260 F F F 21 420-439 156 156 156 261 261 261 F F F 22 440-459 149 149 149 247 247 247 F F F 23 460-479 154 154 154 258 258 258 F F F 24 480-499 156 156 156 261 261 261 F T T <--- 25 500-519 154 154 154 259 259 259 F T T <--- 26 520-539 156 156 156 261 261 261 F F F 27 540-559 154 154 154 259 259 259 F T T <--- 28 560-579 156 156 156 261 261 261 F T T <--- 29 580-599 152 152 152 252 252 252 F F F Max gen: 156, Max flush: 261 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 79 Verifying data now No change after verify Loop: 79 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 261 261 261 F F F 1 020-039 157 157 157 262 262 262 F F F 2 040-059 157 157 157 262 262 262 F F F 3 060-079 157 157 157 262 262 262 F F F 4 080-099 157 157 157 262 262 262 F F F 5 100-119 157 157 157 262 262 262 F F F 6 120-139 150 150 150 248 248 248 F F F 7 140-159 152 152 152 252 252 252 F F F 8 160-179 156 156 156 261 261 261 F F F 9 180-199 157 157 157 262 262 262 F F F 10 200-219 157 157 157 262 262 262 F F F 11 220-239 148 148 148 245 245 245 F F F 12 240-259 156 156 156 261 261 261 F F F 13 260-279 156 156 156 261 261 261 F F F 14 280-299 157 157 157 262 262 262 F F F 15 300-319 157 157 157 262 262 262 F F F 16 320-339 157 157 157 262 262 262 F F F 17 340-359 157 157 157 262 262 262 F F F 18 360-379 157 157 157 262 262 262 F F F 19 380-399 157 157 157 262 262 262 F F F 20 400-419 155 155 155 260 260 260 F F F 21 420-439 156 156 156 261 261 261 F F F 22 440-459 149 149 149 247 247 247 F F F 23 460-479 154 154 154 258 258 258 F F F 24 480-499 157 157 157 262 262 262 F F F 25 500-519 157 157 157 262 262 262 F F F 26 520-539 156 156 156 261 261 261 F F F 27 540-559 157 157 157 262 262 262 F F F 28 560-579 157 157 157 262 262 262 F F F 29 580-599 152 152 152 252 252 252 F F F Max gen: 157, Max flush: 262 Begin loop 80 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:42.816962247Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1773,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:42.825131361Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1773,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:42.825425803Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1773} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:42.825449252Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1773} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:42.825457652Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1773} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:42.825468961Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1773} {"msg":"Crucible efe73724-4c23-4a33-8684-d1c9d1b77974 has session id: d25674d6-5fe0-4c57-9461-30884e94f588","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:42.825828921Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1773,"session_id":"d25674d6-5fe0-4c57-9461-30884e94f588"} {"msg":"Upstairs opts: Upstairs UUID: efe73724-4c23-4a33-8684-d1c9d1b77974, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:42.82585806Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1773,"session_id":"d25674d6-5fe0-4c57-9461-30884e94f588"} {"msg":"Crucible stats registered with UUID: efe73724-4c23-4a33-8684-d1c9d1b77974","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:42.82586931Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1773,"session_id":"d25674d6-5fe0-4c57-9461-30884e94f588"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:47:42.8258786Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1773,"session_id":"d25674d6-5fe0-4c57-9461-30884e94f588"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:42.826010716Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1773,"":"io task","client":"0","":"downstairs","session_id":"d25674d6-5fe0-4c57-9461-30884e94f588"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:42.826028215Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1773,"":"io task","client":"1","":"downstairs","session_id":"d25674d6-5fe0-4c57-9461-30884e94f588"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:42.826038705Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1773,"":"io task","client":"2","":"downstairs","session_id":"d25674d6-5fe0-4c57-9461-30884e94f588"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:42.826187111Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1773,"task":"crutest"} {"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:42.82619886Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1773} {"msg":"efe73724-4c23-4a33-8684-d1c9d1b77974 active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:42.82620969Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1773,"session_id":"d25674d6-5fe0-4c57-9461-30884e94f588"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:42.82621853Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1773,"client":"0","":"downstairs","session_id":"d25674d6-5fe0-4c57-9461-30884e94f588"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:42.826228219Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1773,"client":"1","":"downstairs","session_id":"d25674d6-5fe0-4c57-9461-30884e94f588"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:42.826237779Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1773,"client":"2","":"downstairs","session_id":"d25674d6-5fe0-4c57-9461-30884e94f588"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:42.826247409Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1773,"":"io task","client":"2","":"downstairs","session_id":"d25674d6-5fe0-4c57-9461-30884e94f588"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:42.826257559Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1773,"":"io task","client":"0","":"downstairs","session_id":"d25674d6-5fe0-4c57-9461-30884e94f588"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:42.826267508Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1773,"":"io task","client":"1","":"downstairs","session_id":"d25674d6-5fe0-4c57-9461-30884e94f588"} {"msg":"ds_connection connected from Ok(127.0.0.1:58429)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:42.826361956Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1773,"":"io task","client":"2","":"downstairs","session_id":"d25674d6-5fe0-4c57-9461-30884e94f588"} {"msg":"ds_connection connected from Ok(127.0.0.1:56129)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:42.826390125Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1773,"":"io task","client":"0","":"downstairs","session_id":"d25674d6-5fe0-4c57-9461-30884e94f588"} {"msg":"ds_connection connected from Ok(127.0.0.1:44680)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:42.826401654Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1773,"":"io task","client":"1","":"downstairs","session_id":"d25674d6-5fe0-4c57-9461-30884e94f588"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:42.827038416Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1773,"client":"2","":"downstairs","session_id":"d25674d6-5fe0-4c57-9461-30884e94f588"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:42.827063645Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1773,"client":"1","":"downstairs","session_id":"d25674d6-5fe0-4c57-9461-30884e94f588"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:42.827075105Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1773,"client":"0","":"downstairs","session_id":"d25674d6-5fe0-4c57-9461-30884e94f588"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:42.827337727Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1773,"session_id":"d25674d6-5fe0-4c57-9461-30884e94f588"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:42.827369777Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1773,"session_id":"d25674d6-5fe0-4c57-9461-30884e94f588"} {"msg":"[0]R flush_numbers[0..12]: [261, 262, 262, 262, 262, 262, 248, 252, 261, 262, 262, 245]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:42.827423775Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1773,"":"downstairs","":"downstairs","session_id":"d25674d6-5fe0-4c57-9461-30884e94f588"} {"msg":"[0]R generation[0..12]: [156, 157, 157, 157, 157, 157, 150, 152, 156, 157, 157, 148]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:42.827435005Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1773,"":"downstairs","":"downstairs","session_id":"d25674d6-5fe0-4c57-9461-30884e94f588"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:42.827445924Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1773,"":"downstairs","":"downstairs","session_id":"d25674d6-5fe0-4c57-9461-30884e94f588"} {"msg":"[1]R flush_numbers[0..12]: [261, 262, 262, 262, 262, 262, 248, 252, 261, 262, 262, 245]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:42.827456494Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1773,"":"downstairs","":"downstairs","session_id":"d25674d6-5fe0-4c57-9461-30884e94f588"} {"msg":"[1]R generation[0..12]: [156, 157, 157, 157, 157, 157, 150, 152, 156, 157, 157, 148]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:42.827467024Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1773,"":"downstairs","":"downstairs","session_id":"d25674d6-5fe0-4c57-9461-30884e94f588"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:42.827477803Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1773,"":"downstairs","":"downstairs","session_id":"d25674d6-5fe0-4c57-9461-30884e94f588"} {"msg":"[2]R flush_numbers[0..12]: [261, 262, 262, 262, 262, 262, 248, 252, 261, 262, 262, 245]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:42.827488203Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1773,"":"downstairs","":"downstairs","session_id":"d25674d6-5fe0-4c57-9461-30884e94f588"} {"msg":"[2]R generation[0..12]: [156, 157, 157, 157, 157, 157, 150, 152, 156, 157, 157, 148]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:42.827498733Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1773,"":"downstairs","":"downstairs","session_id":"d25674d6-5fe0-4c57-9461-30884e94f588"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:42.827509543Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1773,"":"downstairs","":"downstairs","session_id":"d25674d6-5fe0-4c57-9461-30884e94f588"} {"msg":"Max found gen is 158","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:42.827518872Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1773,"":"downstairs","":"downstairs","session_id":"d25674d6-5fe0-4c57-9461-30884e94f588"} {"msg":"Generation requested: 160 >= found:158","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:42.827528612Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1773,"":"downstairs","":"downstairs","session_id":"d25674d6-5fe0-4c57-9461-30884e94f588"} {"msg":"Next flush: 263","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:42.827537882Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1773,"":"downstairs","":"downstairs","session_id":"d25674d6-5fe0-4c57-9461-30884e94f588"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:42.827547631Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1773,"":"downstairs","":"downstairs","session_id":"d25674d6-5fe0-4c57-9461-30884e94f588"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:42.827559141Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1773,"session_id":"d25674d6-5fe0-4c57-9461-30884e94f588"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:42.827567871Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1773,"client":"0","":"downstairs","session_id":"d25674d6-5fe0-4c57-9461-30884e94f588"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:42.827577611Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1773,"client":"1","":"downstairs","session_id":"d25674d6-5fe0-4c57-9461-30884e94f588"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:42.82758725Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1773,"client":"2","":"downstairs","session_id":"d25674d6-5fe0-4c57-9461-30884e94f588"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:42.8275973Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1773,"session_id":"d25674d6-5fe0-4c57-9461-30884e94f588"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:42.82760657Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1773,"session_id":"d25674d6-5fe0-4c57-9461-30884e94f588"} {"msg":"efe73724-4c23-4a33-8684-d1c9d1b77974 is now active with session: d25674d6-5fe0-4c57-9461-30884e94f588","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:42.827616369Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1773,"session_id":"d25674d6-5fe0-4c57-9461-30884e94f588"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:42.827625119Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1773,"session_id":"d25674d6-5fe0-4c57-9461-30884e94f588"} {"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:42.827634179Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1773} {"msg":"Activated sub_volume efe73724-4c23-4a33-8684-d1c9d1b77974","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:42.827642129Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1773} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:42.827650278Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1773,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:42.827926031Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1773,"task":"crutest"} 01/30 Write block 214 len 24576 data: 14 11 13 13 12 11 02/30 Flush 03/30 Flush 04/30 Write block 237 len 24576 data: 11 9 10 10 14 13 05/30 Write block 140 len 24576 data: 12 15 16 16 16 15 06/30 Read block 533 len 12288 07/30 Write block 106 len 28672 data: 5 6 5 5 6 9 14 08/30 Write block 401 len 40960 data: 15 16 14 15 15 12 11 7 9 12 09/30 Write block 561 len 4096 data: 11 10/30 Read block 564 len 12288 11/30 Write block 24 len 40960 data: 8 8 5 7 6 8 9 11 10 11 12/30 Read block 224 len 28672 13/30 Write block 471 len 24576 data: 8 6 6 10 14 19 14/30 Read block 408 len 24576 15/30 Read block 182 len 4096 16/30 Read block 533 len 4096 17/30 Read block 240 len 12288 18/30 Read block 472 len 20480 19/30 Flush 20/30 Write block 154 len 12288 data: 10 8 9 21/30 Write block 240 len 28672 data: 11 15 14 13 13 13 15 22/30 Write block 345 len 4096 data: 10 23/30 Write block 40 len 8192 data: 7 8 24/30 Write block 575 len 28672 data: 12 14 12 13 12 11 11 25/30 Read block 280 len 12288 26/30 Write block 32 len 40960 data: 11 12 10 9 11 11 9 7 8 9 27/30 Write block 451 len 32768 data: 9 7 7 7 7 9 10 9 28/30 Write block 156 len 28672 data: 10 9 14 15 13 9 8 29/30 Write block 156 len 8192 data: 11 10 30/30 Write block 93 len 12288 data: 10 12 14 ---------------------------------------------------------------- Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Sent Done false 1001 Acked Flush 0 Done Sent Done false 1002 Acked Flush 0 Done Sent Done false 1003 Acked Write 6 Done Sent Done false 1004 Acked Write 6 Done Sent Done false 1005 Acked Read 3 Done Sent Done false 1006 Acked Write 7 Done Sent Done false 1007 Acked Write 10 Done Sent Done false 1008 Acked Write 1 Done Sent Done false 1009 Acked Read 3 Done Sent Done false 1010 Acked Write 10 Done Sent Done false 1011 Acked Read 7 Done Sent Done false 1012 Acked Write 6 Done Sent Done false 1013 Acked Read 6 Done Sent Done false 1014 Acked Read 1 Done Sent Done false 1015 Acked Read 1 Done Sent Done false 1016 Acked Read 3 Done Sent Done false 1017 Acked Read 5 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Write 3 Done Sent Done false 1020 Acked Write 7 Done Sent Done false 1021 Acked Write 1 Done Sent Done false 1022 Acked Write 2 Done Sent Done false 1023 Acked Write 7 Done Sent Done false 1024 Acked Read 3 Done Sent Done false 1025 Acked Write 10 Done Sent Done false 1026 Acked Write 8 Sent Sent Sent false 1027 Acked Write 7 Sent Sent Sent false 1028 Acked Write 2 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 4 30 4 38 Done 26 0 26 52 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) None Some(JobId(1018)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:47:42.835799323Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1773} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:42.836002918Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1773,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 261 261 261 F F F 1 020-039 160 157 160 265 262 265 T F T <--- 2 040-059 157 157 157 262 262 262 T F T <--- 3 060-079 157 157 157 262 262 262 F F F 4 080-099 157 157 157 262 262 262 T F T <--- 5 100-119 160 157 160 265 262 265 F F F <--- 6 120-139 150 150 150 248 248 248 F F F 7 140-159 160 152 160 265 252 265 T F T <--- 8 160-179 156 156 156 261 261 261 T F T <--- 9 180-199 157 157 157 262 262 262 F F F 10 200-219 160 157 160 263 262 263 F F F <--- 11 220-239 160 148 160 265 245 265 F F F <--- 12 240-259 160 156 160 265 261 265 T F T <--- 13 260-279 156 156 156 261 261 261 F F F 14 280-299 157 157 157 262 262 262 F F F 15 300-319 157 157 157 262 262 262 F F F 16 320-339 157 157 157 262 262 262 F F F 17 340-359 157 157 157 262 262 262 T F T <--- 18 360-379 157 157 157 262 262 262 F F F 19 380-399 157 157 157 262 262 262 F F F 20 400-419 160 155 160 265 260 265 F F F <--- 21 420-439 156 156 156 261 261 261 F F F 22 440-459 149 149 149 247 247 247 T F T <--- 23 460-479 160 154 160 265 258 265 F F F <--- 24 480-499 157 157 157 262 262 262 F F F 25 500-519 157 157 157 262 262 262 F F F 26 520-539 156 156 156 261 261 261 F F F 27 540-559 157 157 157 262 262 262 F F F 28 560-579 160 157 160 265 262 265 T F T <--- 29 580-599 152 152 152 252 252 252 T F T <--- Max gen: 160, Max flush: 265 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 80 Verifying data now No change after verify Loop: 80 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 261 261 261 F F F 1 020-039 161 161 161 266 266 266 F F F 2 040-059 161 161 161 266 266 266 F F F 3 060-079 157 157 157 262 262 262 F F F 4 080-099 161 161 161 266 266 266 F F F 5 100-119 160 160 160 265 265 265 F F F 6 120-139 150 150 150 248 248 248 F F F 7 140-159 161 161 161 266 266 266 F F F 8 160-179 161 161 161 266 266 266 F F F 9 180-199 157 157 157 262 262 262 F F F 10 200-219 160 160 160 263 263 263 F F F 11 220-239 160 160 160 265 265 265 F F F 12 240-259 161 161 161 266 266 266 F F F 13 260-279 156 156 156 261 261 261 F F F 14 280-299 157 157 157 262 262 262 F F F 15 300-319 157 157 157 262 262 262 F F F 16 320-339 157 157 157 262 262 262 F F F 17 340-359 161 161 161 266 266 266 F F F 18 360-379 157 157 157 262 262 262 F F F 19 380-399 157 157 157 262 262 262 F F F 20 400-419 160 160 160 265 265 265 F F F 21 420-439 156 156 156 261 261 261 F F F 22 440-459 161 161 161 266 266 266 F F F 23 460-479 160 160 160 265 265 265 F F F 24 480-499 157 157 157 262 262 262 F F F 25 500-519 157 157 157 262 262 262 F F F 26 520-539 156 156 156 261 261 261 F F F 27 540-559 157 157 157 262 262 262 F F F 28 560-579 161 161 161 266 266 266 F F F 29 580-599 161 161 161 266 266 266 F F F Max gen: 161, Max flush: 266 Begin loop 81 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:47.087122406Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1781,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:47.095151154Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1781,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:47.095445035Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1781} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:47.095491194Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1781} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:47.095519473Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1781} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:47.095531943Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1781} {"msg":"Crucible 240ba4e5-5380-471d-af31-8761530eabd2 has session id: 68596c3c-4b08-4d8b-9599-bdfc72256a3c","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:47.095823495Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1781,"session_id":"68596c3c-4b08-4d8b-9599-bdfc72256a3c"} {"msg":"Upstairs opts: Upstairs UUID: 240ba4e5-5380-471d-af31-8761530eabd2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:47.095852164Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1781,"session_id":"68596c3c-4b08-4d8b-9599-bdfc72256a3c"} {"msg":"Crucible stats registered with UUID: 240ba4e5-5380-471d-af31-8761530eabd2","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:47.095862783Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1781,"session_id":"68596c3c-4b08-4d8b-9599-bdfc72256a3c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:47:47.095871753Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1781,"session_id":"68596c3c-4b08-4d8b-9599-bdfc72256a3c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:47.09599829Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1781,"":"io task","client":"0","":"downstairs","session_id":"68596c3c-4b08-4d8b-9599-bdfc72256a3c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:47.096016389Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1781,"":"io task","client":"1","":"downstairs","session_id":"68596c3c-4b08-4d8b-9599-bdfc72256a3c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:47.096026439Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1781,"":"io task","client":"2","":"downstairs","session_id":"68596c3c-4b08-4d8b-9599-bdfc72256a3c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:47.096188094Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1781,"task":"crutest"} {"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:47.096203534Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1781} {"msg":"240ba4e5-5380-471d-af31-8761530eabd2 active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:47.096228053Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1781,"session_id":"68596c3c-4b08-4d8b-9599-bdfc72256a3c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:47.096237453Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1781,"client":"0","":"downstairs","session_id":"68596c3c-4b08-4d8b-9599-bdfc72256a3c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:47.096247052Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1781,"client":"1","":"downstairs","session_id":"68596c3c-4b08-4d8b-9599-bdfc72256a3c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:47.096258732Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1781,"client":"2","":"downstairs","session_id":"68596c3c-4b08-4d8b-9599-bdfc72256a3c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:47.096268442Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1781,"":"io task","client":"2","":"downstairs","session_id":"68596c3c-4b08-4d8b-9599-bdfc72256a3c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:47.096295181Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1781,"":"io task","client":"0","":"downstairs","session_id":"68596c3c-4b08-4d8b-9599-bdfc72256a3c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:47.09631874Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1781,"":"io task","client":"1","":"downstairs","session_id":"68596c3c-4b08-4d8b-9599-bdfc72256a3c"} {"msg":"ds_connection connected from Ok(127.0.0.1:35824)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:47.09633129Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1781,"":"io task","client":"2","":"downstairs","session_id":"68596c3c-4b08-4d8b-9599-bdfc72256a3c"} {"msg":"ds_connection connected from Ok(127.0.0.1:45558)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:47.096346829Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1781,"":"io task","client":"1","":"downstairs","session_id":"68596c3c-4b08-4d8b-9599-bdfc72256a3c"} {"msg":"ds_connection connected from Ok(127.0.0.1:47673)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:47.096445267Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1781,"":"io task","client":"0","":"downstairs","session_id":"68596c3c-4b08-4d8b-9599-bdfc72256a3c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:47.096935792Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1781,"client":"1","":"downstairs","session_id":"68596c3c-4b08-4d8b-9599-bdfc72256a3c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:47.096990741Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1781,"client":"0","":"downstairs","session_id":"68596c3c-4b08-4d8b-9599-bdfc72256a3c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:47.097094708Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1781,"client":"2","":"downstairs","session_id":"68596c3c-4b08-4d8b-9599-bdfc72256a3c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:47.097222354Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1781,"session_id":"68596c3c-4b08-4d8b-9599-bdfc72256a3c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:47.097314482Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1781,"session_id":"68596c3c-4b08-4d8b-9599-bdfc72256a3c"} {"msg":"[0]R flush_numbers[0..12]: [261, 266, 266, 262, 266, 265, 248, 266, 266, 262, 263, 265]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:47.09737761Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1781,"":"downstairs","":"downstairs","session_id":"68596c3c-4b08-4d8b-9599-bdfc72256a3c"} {"msg":"[0]R generation[0..12]: [156, 161, 161, 157, 161, 160, 150, 161, 161, 157, 160, 160]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:47.097389009Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1781,"":"downstairs","":"downstairs","session_id":"68596c3c-4b08-4d8b-9599-bdfc72256a3c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:47.097399999Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1781,"":"downstairs","":"downstairs","session_id":"68596c3c-4b08-4d8b-9599-bdfc72256a3c"} {"msg":"[1]R flush_numbers[0..12]: [261, 266, 266, 262, 266, 265, 248, 266, 266, 262, 263, 265]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:47.097413739Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1781,"":"downstairs","":"downstairs","session_id":"68596c3c-4b08-4d8b-9599-bdfc72256a3c"} {"msg":"[1]R generation[0..12]: [156, 161, 161, 157, 161, 160, 150, 161, 161, 157, 160, 160]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:47.097424438Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1781,"":"downstairs","":"downstairs","session_id":"68596c3c-4b08-4d8b-9599-bdfc72256a3c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:47.097435288Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1781,"":"downstairs","":"downstairs","session_id":"68596c3c-4b08-4d8b-9599-bdfc72256a3c"} {"msg":"[2]R flush_numbers[0..12]: [261, 266, 266, 262, 266, 265, 248, 266, 266, 262, 263, 265]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:47.097445828Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1781,"":"downstairs","":"downstairs","session_id":"68596c3c-4b08-4d8b-9599-bdfc72256a3c"} {"msg":"[2]R generation[0..12]: [156, 161, 161, 157, 161, 160, 150, 161, 161, 157, 160, 160]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:47.097456367Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1781,"":"downstairs","":"downstairs","session_id":"68596c3c-4b08-4d8b-9599-bdfc72256a3c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:47.097467237Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1781,"":"downstairs","":"downstairs","session_id":"68596c3c-4b08-4d8b-9599-bdfc72256a3c"} {"msg":"Max found gen is 162","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:47.097476547Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1781,"":"downstairs","":"downstairs","session_id":"68596c3c-4b08-4d8b-9599-bdfc72256a3c"} {"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:47.097486307Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1781,"":"downstairs","":"downstairs","session_id":"68596c3c-4b08-4d8b-9599-bdfc72256a3c"} {"msg":"Next flush: 267","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:47.097495636Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1781,"":"downstairs","":"downstairs","session_id":"68596c3c-4b08-4d8b-9599-bdfc72256a3c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:47.097505046Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1781,"":"downstairs","":"downstairs","session_id":"68596c3c-4b08-4d8b-9599-bdfc72256a3c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:47.097514886Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1781,"session_id":"68596c3c-4b08-4d8b-9599-bdfc72256a3c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:47.097523536Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1781,"client":"0","":"downstairs","session_id":"68596c3c-4b08-4d8b-9599-bdfc72256a3c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:47.097533135Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1781,"client":"1","":"downstairs","session_id":"68596c3c-4b08-4d8b-9599-bdfc72256a3c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:47.097542725Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1781,"client":"2","":"downstairs","session_id":"68596c3c-4b08-4d8b-9599-bdfc72256a3c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:47.097552405Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1781,"session_id":"68596c3c-4b08-4d8b-9599-bdfc72256a3c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:47.097562934Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1781,"session_id":"68596c3c-4b08-4d8b-9599-bdfc72256a3c"} {"msg":"240ba4e5-5380-471d-af31-8761530eabd2 is now active with session: 68596c3c-4b08-4d8b-9599-bdfc72256a3c","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:47.097572814Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1781,"session_id":"68596c3c-4b08-4d8b-9599-bdfc72256a3c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:47.097581464Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1781,"session_id":"68596c3c-4b08-4d8b-9599-bdfc72256a3c"} {"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:47.097590594Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1781} {"msg":"Activated sub_volume 240ba4e5-5380-471d-af31-8761530eabd2","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:47.097598553Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1781} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:47.097606643Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1781,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:47.097878655Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1781,"task":"crutest"} 01/30 Flush 02/30 Write block 468 len 40960 data: 8 9 10 9 7 7 11 15 20 19 03/30 Read block 503 len 32768 04/30 Write block 356 len 16384 data: 13 13 12 11 05/30 Read block 55 len 16384 06/30 Read block 47 len 28672 07/30 Read block 10 len 36864 08/30 Read block 112 len 8192 09/30 Flush 10/30 Write block 507 len 32768 data: 18 12 11 12 18 13 9 9 11/30 Read block 2 len 20480 12/30 Read block 365 len 12288 13/30 Read block 292 len 20480 14/30 Flush 15/30 Flush 16/30 Write block 549 len 32768 data: 15 13 8 12 14 11 11 9 17/30 Write block 375 len 8192 data: 11 16 18/30 Write block 529 len 4096 data: 7 19/30 Write block 359 len 16384 data: 12 13 15 18 20/30 Write block 276 len 12288 data: 11 10 11 21/30 Read block 450 len 8192 22/30 Read block 440 len 24576 23/30 Read block 37 len 28672 24/30 Read block 30 len 8192 25/30 Read block 144 len 8192 26/30 Write block 452 len 32768 data: 8 8 8 8 10 11 10 7 27/30 Write block 317 len 24576 data: 10 6 8 7 11 11 28/30 Read block 0 len 12288 29/30 Read block 123 len 20480 30/30 Write block 269 len 20480 data: 8 8 11 9 11 ---------------------------------------------------------------- Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 10 Done Done Done false 1002 Acked Read 8 Done Done Done false 1003 Acked Write 4 Done Done Done false 1004 Acked Read 4 Done Done Done false 1005 Acked Read 7 Done Done Sent false 1006 Acked Read 9 Done Done Sent false 1007 Acked Read 2 Done Done Sent false 1008 Acked Flush 0 Done Done Sent false 1009 Acked Write 8 Done Done Sent false 1010 Acked Read 5 Done Done Sent false 1011 Acked Read 3 Done Done Sent false 1012 Acked Read 5 Done Done Sent false 1013 Acked Flush 0 Done Done Sent false 1014 Acked Flush 0 Done Done Sent false 1015 Acked Write 8 Done Done Sent false 1016 Acked Write 2 Done Done Sent false 1017 Acked Write 1 Done Done Sent false 1018 Acked Write 4 Done Done Sent false 1019 Acked Write 3 Done Done Sent false 1020 Acked Read 2 Done Done Sent false 1021 Acked Read 6 Done Done Sent false 1022 Acked Read 7 Done Done Sent false 1023 Acked Read 2 Done Done Sent false 1024 Acked Read 2 Done Done Sent false 1025 Acked Write 8 Done Done Sent false 1026 Acked Write 6 Done Done Sent false 1027 Acked Read 3 Done Done Sent false 1028 Acked Read 5 Done Done Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 25 27 Done 28 28 4 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) Some(JobId(1014)) Some(JobId(1000)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:47:47.106364321Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1781} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:47.106555265Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1781,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 261 261 261 F F F 1 020-039 161 161 161 266 266 266 F F F 2 040-059 161 161 161 266 266 266 F F F 3 060-079 157 157 157 262 262 262 F F F 4 080-099 161 161 161 266 266 266 F F F 5 100-119 160 160 160 265 265 265 F F F 6 120-139 150 150 150 248 248 248 F F F 7 140-159 161 161 161 266 266 266 F F F 8 160-179 161 161 161 266 266 266 F F F 9 180-199 157 157 157 262 262 262 F F F 10 200-219 160 160 160 263 263 263 F F F 11 220-239 160 160 160 265 265 265 F F F 12 240-259 161 161 161 266 266 266 F F F 13 260-279 156 156 156 261 261 261 T T F <--- 14 280-299 157 157 157 262 262 262 F F F 15 300-319 157 157 157 262 262 262 T T F <--- 16 320-339 157 157 157 262 262 262 T T F <--- 17 340-359 162 162 161 268 268 266 T T T <--- 18 360-379 157 157 157 262 262 262 T T F <--- 19 380-399 157 157 157 262 262 262 F F F 20 400-419 160 160 160 265 265 265 F F F 21 420-439 156 156 156 261 261 261 F F F 22 440-459 161 161 161 266 266 266 T T F <--- 23 460-479 162 162 160 268 268 265 F F T <--- 24 480-499 157 157 157 262 262 262 F F F 25 500-519 162 162 157 269 269 262 F F F <--- 26 520-539 156 156 156 261 261 261 T T F <--- 27 540-559 157 157 157 262 262 262 T T F <--- 28 560-579 161 161 161 266 266 266 F F F 29 580-599 161 161 161 266 266 266 F F F Max gen: 162, Max flush: 269 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 81 Verifying data now No change after verify Loop: 81 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 261 261 261 F F F 1 020-039 161 161 161 266 266 266 F F F 2 040-059 161 161 161 266 266 266 F F F 3 060-079 157 157 157 262 262 262 F F F 4 080-099 161 161 161 266 266 266 F F F 5 100-119 160 160 160 265 265 265 F F F 6 120-139 150 150 150 248 248 248 F F F 7 140-159 161 161 161 266 266 266 F F F 8 160-179 161 161 161 266 266 266 F F F 9 180-199 157 157 157 262 262 262 F F F 10 200-219 160 160 160 263 263 263 F F F 11 220-239 160 160 160 265 265 265 F F F 12 240-259 161 161 161 266 266 266 F F F 13 260-279 163 163 163 270 270 270 F F F 14 280-299 157 157 157 262 262 262 F F F 15 300-319 163 163 163 270 270 270 F F F 16 320-339 163 163 163 270 270 270 F F F 17 340-359 163 163 163 270 270 270 F F F 18 360-379 163 163 163 270 270 270 F F F 19 380-399 157 157 157 262 262 262 F F F 20 400-419 160 160 160 265 265 265 F F F 21 420-439 156 156 156 261 261 261 F F F 22 440-459 163 163 163 270 270 270 F F F 23 460-479 162 162 162 268 268 268 F F F 24 480-499 157 157 157 262 262 262 F F F 25 500-519 162 162 162 269 269 269 F F F 26 520-539 163 163 163 270 270 270 F F F 27 540-559 163 163 163 270 270 270 F F F 28 560-579 161 161 161 266 266 266 F F F 29 580-599 161 161 161 266 266 266 F F F Max gen: 163, Max flush: 270 Begin loop 82 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:51.351868901Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1789,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:47:51.358886158Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1789,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:51.358936117Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1789,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:51.365728901Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1789,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:51.365992903Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1789} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:51.366022612Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1789} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:51.366031082Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1789} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:51.366042522Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1789} {"msg":"Crucible b2c4b3d1-90b4-4d08-840f-c10b2203e61d has session id: 28ce752a-fb57-459c-a493-a436c095eff7","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:51.366328743Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1789,"session_id":"28ce752a-fb57-459c-a493-a436c095eff7"} {"msg":"Upstairs opts: Upstairs UUID: b2c4b3d1-90b4-4d08-840f-c10b2203e61d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:51.366363742Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1789,"session_id":"28ce752a-fb57-459c-a493-a436c095eff7"} {"msg":"Crucible stats registered with UUID: b2c4b3d1-90b4-4d08-840f-c10b2203e61d","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:51.366374352Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1789,"session_id":"28ce752a-fb57-459c-a493-a436c095eff7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:47:51.366383502Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1789,"session_id":"28ce752a-fb57-459c-a493-a436c095eff7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:51.366520338Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1789,"":"io task","client":"1","":"downstairs","session_id":"28ce752a-fb57-459c-a493-a436c095eff7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:51.366537477Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1789,"":"io task","client":"0","":"downstairs","session_id":"28ce752a-fb57-459c-a493-a436c095eff7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:51.366547397Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1789,"":"io task","client":"2","":"downstairs","session_id":"28ce752a-fb57-459c-a493-a436c095eff7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:51.366689993Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1789,"task":"crutest"} {"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:51.366704593Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1789} {"msg":"b2c4b3d1-90b4-4d08-840f-c10b2203e61d active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:51.366732162Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1789,"session_id":"28ce752a-fb57-459c-a493-a436c095eff7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:51.366741442Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1789,"client":"0","":"downstairs","session_id":"28ce752a-fb57-459c-a493-a436c095eff7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:51.366751071Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1789,"client":"1","":"downstairs","session_id":"28ce752a-fb57-459c-a493-a436c095eff7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:51.366760771Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1789,"client":"2","":"downstairs","session_id":"28ce752a-fb57-459c-a493-a436c095eff7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:51.366770231Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1789,"":"io task","client":"2","":"downstairs","session_id":"28ce752a-fb57-459c-a493-a436c095eff7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:51.36678013Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1789,"":"io task","client":"0","":"downstairs","session_id":"28ce752a-fb57-459c-a493-a436c095eff7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:51.36679039Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1789,"":"io task","client":"1","":"downstairs","session_id":"28ce752a-fb57-459c-a493-a436c095eff7"} {"msg":"ds_connection connected from Ok(127.0.0.1:35327)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:51.366834459Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1789,"":"io task","client":"1","":"downstairs","session_id":"28ce752a-fb57-459c-a493-a436c095eff7"} {"msg":"ds_connection connected from Ok(127.0.0.1:48999)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:51.366959585Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1789,"":"io task","client":"0","":"downstairs","session_id":"28ce752a-fb57-459c-a493-a436c095eff7"} {"msg":"ds_connection connected from Ok(127.0.0.1:43655)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:51.366985784Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1789,"":"io task","client":"2","":"downstairs","session_id":"28ce752a-fb57-459c-a493-a436c095eff7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:51.367469941Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1789,"client":"2","":"downstairs","session_id":"28ce752a-fb57-459c-a493-a436c095eff7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:51.36749382Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1789,"client":"1","":"downstairs","session_id":"28ce752a-fb57-459c-a493-a436c095eff7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:51.367583317Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1789,"client":"0","":"downstairs","session_id":"28ce752a-fb57-459c-a493-a436c095eff7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:51.367762642Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1789,"session_id":"28ce752a-fb57-459c-a493-a436c095eff7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:51.367779572Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1789,"session_id":"28ce752a-fb57-459c-a493-a436c095eff7"} {"msg":"[0]R flush_numbers[0..12]: [261, 266, 266, 262, 266, 265, 248, 266, 266, 262, 263, 265]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:51.36785075Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1789,"":"downstairs","":"downstairs","session_id":"28ce752a-fb57-459c-a493-a436c095eff7"} {"msg":"[0]R generation[0..12]: [156, 161, 161, 157, 161, 160, 150, 161, 161, 157, 160, 160]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:51.367862089Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1789,"":"downstairs","":"downstairs","session_id":"28ce752a-fb57-459c-a493-a436c095eff7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:51.367873099Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1789,"":"downstairs","":"downstairs","session_id":"28ce752a-fb57-459c-a493-a436c095eff7"} {"msg":"[1]R flush_numbers[0..12]: [261, 266, 266, 262, 266, 265, 248, 266, 266, 262, 263, 265]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:51.367883569Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1789,"":"downstairs","":"downstairs","session_id":"28ce752a-fb57-459c-a493-a436c095eff7"} {"msg":"[1]R generation[0..12]: [156, 161, 161, 157, 161, 160, 150, 161, 161, 157, 160, 160]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:51.367894018Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1789,"":"downstairs","":"downstairs","session_id":"28ce752a-fb57-459c-a493-a436c095eff7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:51.367904948Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1789,"":"downstairs","":"downstairs","session_id":"28ce752a-fb57-459c-a493-a436c095eff7"} {"msg":"[2]R flush_numbers[0..12]: [261, 266, 266, 262, 266, 265, 248, 266, 266, 262, 263, 265]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:51.367917528Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1789,"":"downstairs","":"downstairs","session_id":"28ce752a-fb57-459c-a493-a436c095eff7"} {"msg":"[2]R generation[0..12]: [156, 161, 161, 157, 161, 160, 150, 161, 161, 157, 160, 160]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:51.367928097Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1789,"":"downstairs","":"downstairs","session_id":"28ce752a-fb57-459c-a493-a436c095eff7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:51.367938837Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1789,"":"downstairs","":"downstairs","session_id":"28ce752a-fb57-459c-a493-a436c095eff7"} {"msg":"Max found gen is 164","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:51.367948347Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1789,"":"downstairs","":"downstairs","session_id":"28ce752a-fb57-459c-a493-a436c095eff7"} {"msg":"Generation requested: 164 >= found:164","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:51.367958046Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1789,"":"downstairs","":"downstairs","session_id":"28ce752a-fb57-459c-a493-a436c095eff7"} {"msg":"Next flush: 271","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:51.367967276Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1789,"":"downstairs","":"downstairs","session_id":"28ce752a-fb57-459c-a493-a436c095eff7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:51.367976606Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1789,"":"downstairs","":"downstairs","session_id":"28ce752a-fb57-459c-a493-a436c095eff7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:51.367986316Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1789,"session_id":"28ce752a-fb57-459c-a493-a436c095eff7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:51.367994925Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1789,"client":"0","":"downstairs","session_id":"28ce752a-fb57-459c-a493-a436c095eff7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:51.368004455Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1789,"client":"1","":"downstairs","session_id":"28ce752a-fb57-459c-a493-a436c095eff7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:51.368014195Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1789,"client":"2","":"downstairs","session_id":"28ce752a-fb57-459c-a493-a436c095eff7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:51.368023815Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1789,"session_id":"28ce752a-fb57-459c-a493-a436c095eff7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:51.368032874Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1789,"session_id":"28ce752a-fb57-459c-a493-a436c095eff7"} {"msg":"b2c4b3d1-90b4-4d08-840f-c10b2203e61d is now active with session: 28ce752a-fb57-459c-a493-a436c095eff7","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:51.368042684Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1789,"session_id":"28ce752a-fb57-459c-a493-a436c095eff7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:51.368051264Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1789,"session_id":"28ce752a-fb57-459c-a493-a436c095eff7"} {"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:51.368060423Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1789} {"msg":"Activated sub_volume b2c4b3d1-90b4-4d08-840f-c10b2203e61d","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:51.368070023Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1789} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:51.368078043Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1789,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:51.368387204Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1789,"task":"crutest"} 01/30 Write block 559 len 16384 data: 14 14 12 12 02/30 Read block 61 len 20480 03/30 Read block 138 len 32768 04/30 Write block 139 len 24576 data: 11 13 16 17 17 17 05/30 Write block 64 len 4096 data: 8 06/30 Flush 07/30 Write block 319 len 36864 data: 9 8 12 12 14 19 19 20 21 08/30 Read block 528 len 20480 09/30 Read block 61 len 40960 10/30 Read block 533 len 36864 11/30 Write block 211 len 24576 data: 14 15 17 15 12 14 12/30 Write block 589 len 24576 data: 10 10 9 8 8 7 13/30 Read block 424 len 28672 14/30 Write block 453 len 40960 data: 9 9 9 11 12 11 8 6 7 7 15/30 Write block 89 len 12288 data: 10 11 8 16/30 Read block 409 len 4096 17/30 Read block 514 len 36864 18/30 Read block 89 len 20480 19/30 Flush 20/30 Write block 171 len 24576 data: 12 15 17 14 14 12 21/30 Write block 316 len 24576 data: 11 11 7 10 9 13 22/30 Write block 116 len 8192 data: 15 13 23/30 Write block 526 len 32768 data: 8 8 5 8 8 9 15 14 24/30 Read block 190 len 20480 25/30 Read block 127 len 4096 26/30 Read block 68 len 36864 27/30 Read block 495 len 32768 28/30 Write block 539 len 28672 data: 20 20 15 16 16 16 14 29/30 Read block 345 len 16384 30/30 Write block 524 len 28672 data: 13 10 9 9 6 9 9 ---------------------------------------------------------------- Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Read 5 Done Done Done false 1002 Acked Read 8 Sent Done Done false 1003 Acked Write 6 Sent Done Done false 1004 Acked Write 1 Sent Done Done false 1005 Acked Flush 0 Sent Done Done false 1006 Acked Write 9 Sent Done Done false 1007 Acked Read 5 Sent Done Done false 1008 Acked Read 10 Sent Done Done false 1009 Acked Read 9 Sent Done Done false 1010 Acked Write 6 Sent Done Done false 1011 Acked Write 6 Sent Done Done false 1012 Acked Read 7 Sent Done Done false 1013 Acked Write 10 Sent Done Done false 1014 Acked Write 3 Sent Done Done false 1015 Acked Read 1 Sent Done Done false 1016 Acked Read 9 Sent Done Done false 1017 Acked Read 5 Sent Done Done false 1018 Acked Flush 0 Sent Done Done false 1019 Acked Write 6 Sent Done Done false 1020 Acked Write 6 Sent Done Done false 1021 Acked Write 2 Sent Done Done false 1022 Acked Write 8 Sent Done Done false 1023 Acked Read 5 Sent Done Done false 1024 Acked Read 1 Sent Done Done false 1025 Acked Read 9 Sent Done Done false 1026 Acked Read 8 Sent Done Done false 1027 Acked Write 7 Sent Done Done false 1028 Acked Read 4 Sent Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 1 1 30 Done 2 29 29 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:47:51.377563169Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1789} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:51.377745614Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1789,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 261 261 261 F F F 1 020-039 161 161 161 266 266 266 F F F 2 040-059 161 161 161 266 266 266 F F F 3 060-079 157 164 164 262 271 271 F F F <--- 4 080-099 161 164 164 266 272 272 F F F <--- 5 100-119 160 160 160 265 265 265 F T T <--- 6 120-139 150 164 164 248 271 271 F F F <--- 7 140-159 161 164 164 266 271 271 F F F <--- 8 160-179 161 161 161 266 266 266 F T T <--- 9 180-199 157 157 157 262 262 262 F F F 10 200-219 160 164 164 263 272 272 F F F <--- 11 220-239 160 160 160 265 265 265 F F F 12 240-259 161 161 161 266 266 266 F F F 13 260-279 163 163 163 270 270 270 F F F 14 280-299 157 157 157 262 262 262 F F F 15 300-319 163 164 164 270 272 272 F T T <--- 16 320-339 163 164 164 270 272 272 F T T <--- 17 340-359 163 163 163 270 270 270 F F F 18 360-379 163 163 163 270 270 270 F F F 19 380-399 157 157 157 262 262 262 F F F 20 400-419 160 160 160 265 265 265 F F F 21 420-439 156 156 156 261 261 261 F F F 22 440-459 163 164 164 270 272 272 F F F <--- 23 460-479 162 164 164 268 272 272 F F F <--- 24 480-499 157 157 157 262 262 262 F F F 25 500-519 162 162 162 269 269 269 F F F 26 520-539 163 163 163 270 270 270 F T T <--- 27 540-559 163 164 164 270 271 271 T T T <--- 28 560-579 161 164 164 266 271 271 T F F <--- 29 580-599 161 164 164 266 272 272 F F F <--- Max gen: 164, Max flush: 272 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 82 Verifying data now No change after verify Loop: 82 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 261 261 261 F F F 1 020-039 161 161 161 266 266 266 F F F 2 040-059 161 161 161 266 266 266 F F F 3 060-079 164 164 164 271 271 271 F F F 4 080-099 164 164 164 272 272 272 F F F 5 100-119 165 165 165 273 273 273 F F F 6 120-139 164 164 164 271 271 271 F F F 7 140-159 164 164 164 271 271 271 F F F 8 160-179 165 165 165 273 273 273 F F F 9 180-199 157 157 157 262 262 262 F F F 10 200-219 164 164 164 272 272 272 F F F 11 220-239 160 160 160 265 265 265 F F F 12 240-259 161 161 161 266 266 266 F F F 13 260-279 163 163 163 270 270 270 F F F 14 280-299 157 157 157 262 262 262 F F F 15 300-319 165 165 165 273 273 273 F F F 16 320-339 165 165 165 273 273 273 F F F 17 340-359 163 163 163 270 270 270 F F F 18 360-379 163 163 163 270 270 270 F F F 19 380-399 157 157 157 262 262 262 F F F 20 400-419 160 160 160 265 265 265 F F F 21 420-439 156 156 156 261 261 261 F F F 22 440-459 164 164 164 272 272 272 F F F 23 460-479 164 164 164 272 272 272 F F F 24 480-499 157 157 157 262 262 262 F F F 25 500-519 162 162 162 269 269 269 F F F 26 520-539 165 165 165 273 273 273 F F F 27 540-559 165 165 165 273 273 273 F F F 28 560-579 164 164 164 271 271 271 F F F 29 580-599 164 164 164 272 272 272 F F F Max gen: 165, Max flush: 273 Begin loop 83 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:55.634094911Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1797,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:55.642245496Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1797,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:55.642563957Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1797} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:55.642589226Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1797} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:55.642597736Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1797} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:55.642609136Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1797} {"msg":"Crucible fb460d7d-804e-4a1e-ba85-2da212035170 has session id: 3db33828-e865-4969-95ee-a05092450106","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:55.642878678Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1797,"session_id":"3db33828-e865-4969-95ee-a05092450106"} {"msg":"Upstairs opts: Upstairs UUID: fb460d7d-804e-4a1e-ba85-2da212035170, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:55.642944376Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1797,"session_id":"3db33828-e865-4969-95ee-a05092450106"} {"msg":"Crucible stats registered with UUID: fb460d7d-804e-4a1e-ba85-2da212035170","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:55.642954386Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1797,"session_id":"3db33828-e865-4969-95ee-a05092450106"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:47:55.642963396Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1797,"session_id":"3db33828-e865-4969-95ee-a05092450106"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:55.643118981Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1797,"":"io task","client":"2","":"downstairs","session_id":"3db33828-e865-4969-95ee-a05092450106"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:55.64315282Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1797,"":"io task","client":"0","":"downstairs","session_id":"3db33828-e865-4969-95ee-a05092450106"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:55.64316373Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1797,"":"io task","client":"1","":"downstairs","session_id":"3db33828-e865-4969-95ee-a05092450106"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:55.643263637Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1797,"task":"crutest"} {"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:55.643274457Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1797} {"msg":"fb460d7d-804e-4a1e-ba85-2da212035170 active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:55.643308086Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1797,"session_id":"3db33828-e865-4969-95ee-a05092450106"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:55.643317475Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1797,"client":"0","":"downstairs","session_id":"3db33828-e865-4969-95ee-a05092450106"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:55.643327075Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1797,"client":"1","":"downstairs","session_id":"3db33828-e865-4969-95ee-a05092450106"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:55.643336615Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1797,"client":"2","":"downstairs","session_id":"3db33828-e865-4969-95ee-a05092450106"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:55.643346264Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1797,"":"io task","client":"2","":"downstairs","session_id":"3db33828-e865-4969-95ee-a05092450106"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:55.643356264Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1797,"":"io task","client":"0","":"downstairs","session_id":"3db33828-e865-4969-95ee-a05092450106"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:55.643366114Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1797,"":"io task","client":"1","":"downstairs","session_id":"3db33828-e865-4969-95ee-a05092450106"} {"msg":"ds_connection connected from Ok(127.0.0.1:51904)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:55.643479891Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1797,"":"io task","client":"2","":"downstairs","session_id":"3db33828-e865-4969-95ee-a05092450106"} {"msg":"ds_connection connected from Ok(127.0.0.1:33944)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:55.643536089Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1797,"":"io task","client":"0","":"downstairs","session_id":"3db33828-e865-4969-95ee-a05092450106"} {"msg":"ds_connection connected from Ok(127.0.0.1:35115)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:55.643547189Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1797,"":"io task","client":"1","":"downstairs","session_id":"3db33828-e865-4969-95ee-a05092450106"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:55.64419816Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1797,"client":"0","":"downstairs","session_id":"3db33828-e865-4969-95ee-a05092450106"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:55.644213139Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1797,"client":"1","":"downstairs","session_id":"3db33828-e865-4969-95ee-a05092450106"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:55.644225999Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1797,"client":"2","":"downstairs","session_id":"3db33828-e865-4969-95ee-a05092450106"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:55.64454834Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1797,"session_id":"3db33828-e865-4969-95ee-a05092450106"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:55.644560969Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1797,"session_id":"3db33828-e865-4969-95ee-a05092450106"} {"msg":"[0]R flush_numbers[0..12]: [261, 266, 266, 271, 272, 273, 271, 271, 273, 262, 272, 265]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:55.644574599Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1797,"":"downstairs","":"downstairs","session_id":"3db33828-e865-4969-95ee-a05092450106"} {"msg":"[0]R generation[0..12]: [156, 161, 161, 164, 164, 165, 164, 164, 165, 157, 164, 160]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:55.644585179Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1797,"":"downstairs","":"downstairs","session_id":"3db33828-e865-4969-95ee-a05092450106"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:55.644596038Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1797,"":"downstairs","":"downstairs","session_id":"3db33828-e865-4969-95ee-a05092450106"} {"msg":"[1]R flush_numbers[0..12]: [261, 266, 266, 271, 272, 273, 271, 271, 273, 262, 272, 265]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:55.644606688Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1797,"":"downstairs","":"downstairs","session_id":"3db33828-e865-4969-95ee-a05092450106"} {"msg":"[1]R generation[0..12]: [156, 161, 161, 164, 164, 165, 164, 164, 165, 157, 164, 160]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:55.644617068Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1797,"":"downstairs","":"downstairs","session_id":"3db33828-e865-4969-95ee-a05092450106"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:55.644627857Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1797,"":"downstairs","":"downstairs","session_id":"3db33828-e865-4969-95ee-a05092450106"} {"msg":"[2]R flush_numbers[0..12]: [261, 266, 266, 271, 272, 273, 271, 271, 273, 262, 272, 265]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:55.644638347Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1797,"":"downstairs","":"downstairs","session_id":"3db33828-e865-4969-95ee-a05092450106"} {"msg":"[2]R generation[0..12]: [156, 161, 161, 164, 164, 165, 164, 164, 165, 157, 164, 160]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:55.644648967Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1797,"":"downstairs","":"downstairs","session_id":"3db33828-e865-4969-95ee-a05092450106"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:55.644659837Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1797,"":"downstairs","":"downstairs","session_id":"3db33828-e865-4969-95ee-a05092450106"} {"msg":"Max found gen is 166","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:55.644669166Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1797,"":"downstairs","":"downstairs","session_id":"3db33828-e865-4969-95ee-a05092450106"} {"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:55.644678926Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1797,"":"downstairs","":"downstairs","session_id":"3db33828-e865-4969-95ee-a05092450106"} {"msg":"Next flush: 274","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:55.644690116Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1797,"":"downstairs","":"downstairs","session_id":"3db33828-e865-4969-95ee-a05092450106"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:55.644699445Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1797,"":"downstairs","":"downstairs","session_id":"3db33828-e865-4969-95ee-a05092450106"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:55.644709125Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1797,"session_id":"3db33828-e865-4969-95ee-a05092450106"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:55.644717825Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1797,"client":"0","":"downstairs","session_id":"3db33828-e865-4969-95ee-a05092450106"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:55.644727405Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1797,"client":"1","":"downstairs","session_id":"3db33828-e865-4969-95ee-a05092450106"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:55.644736874Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1797,"client":"2","":"downstairs","session_id":"3db33828-e865-4969-95ee-a05092450106"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:55.644746594Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1797,"session_id":"3db33828-e865-4969-95ee-a05092450106"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:55.644755714Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1797,"session_id":"3db33828-e865-4969-95ee-a05092450106"} {"msg":"fb460d7d-804e-4a1e-ba85-2da212035170 is now active with session: 3db33828-e865-4969-95ee-a05092450106","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:55.644765404Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1797,"session_id":"3db33828-e865-4969-95ee-a05092450106"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:55.644773993Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1797,"session_id":"3db33828-e865-4969-95ee-a05092450106"} {"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:55.644783153Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1797} {"msg":"Activated sub_volume fb460d7d-804e-4a1e-ba85-2da212035170","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:55.644791083Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1797} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:55.644799113Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1797,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:55.645140873Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1797,"task":"crutest"} 01/30 Write block 78 len 20480 data: 18 16 12 10 11 02/30 Write block 425 len 32768 data: 12 12 12 11 14 11 10 9 03/30 Write block 230 len 24576 data: 16 17 13 14 14 11 04/30 Read block 329 len 40960 05/30 Read block 102 len 24576 06/30 Read block 527 len 40960 07/30 Read block 17 len 40960 08/30 Read block 470 len 24576 09/30 Write block 457 len 36864 data: 13 12 9 7 8 8 8 7 5 10/30 Write block 225 len 32768 data: 17 16 17 15 15 17 18 14 11/30 Write block 11 len 28672 data: 11 8 9 8 9 10 10 12/30 Write block 197 len 4096 data: 12 13/30 Write block 380 len 4096 data: 19 14/30 Write block 102 len 24576 data: 7 7 6 5 6 7 15/30 Read block 492 len 16384 16/30 Write block 155 len 28672 data: 9 12 11 15 16 14 10 17/30 Write block 485 len 20480 data: 14 15 17 19 19 18/30 Read block 508 len 24576 19/30 Read block 509 len 32768 20/30 Write block 534 len 24576 data: 14 18 16 19 21 21 21/30 Flush 22/30 Write block 53 len 16384 data: 10 10 13 13 23/30 Write block 147 len 8192 data: 15 13 24/30 Write block 118 len 36864 data: 12 12 10 11 13 15 13 13 12 25/30 Write block 74 len 8192 data: 16 15 26/30 Read block 299 len 16384 27/30 Write block 256 len 4096 data: 14 28/30 Write block 14 len 8192 data: 9 10 29/30 Read block 140 len 12288 30/30 Write block 512 len 28672 data: 14 10 10 8 8 9 11 ---------------------------------------------------------------- Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Write 8 Done Done Done false 1002 Acked Write 6 Done Done Done false 1003 Acked Read 10 Done Done Done false 1004 Acked Read 6 Done Done Done false 1005 Acked Read 10 Done Sent Done false 1006 Acked Read 10 Done Sent Done false 1007 Acked Read 6 Done Sent Done false 1008 Acked Write 9 Done Sent Done false 1009 Acked Write 8 Done Sent Done false 1010 Acked Write 7 Done Sent Done false 1011 Acked Write 1 Done Sent Done false 1012 Acked Write 1 Done Sent Done false 1013 Acked Write 6 Done Sent Done false 1014 Acked Read 4 Done Sent Done false 1015 Acked Write 7 Done Sent Done false 1016 Acked Write 5 Done Sent Done false 1017 Acked Read 6 Done Sent Done false 1018 Acked Read 8 Done Sent Done false 1019 Acked Write 6 Done Sent Done false 1020 Acked Flush 0 Done Sent Done false 1021 Acked Write 4 Done Sent Done false 1022 Acked Write 2 Done Sent Done false 1023 Acked Write 9 Done Sent Done false 1024 Acked Write 2 Done Sent Done false 1025 Acked Read 4 Done Sent Done false 1026 Acked Write 1 Done Sent Done false 1027 Acked Write 2 Done Sent Done false 1028 Acked Read 3 Done Sent Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 25 1 27 Done 29 5 29 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) None Some(JobId(1020)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:47:55.653050865Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1797} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:55.653244529Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1797,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 166 156 166 274 261 274 T F T <--- 1 020-039 161 161 161 266 266 266 F F F 2 040-059 161 161 161 266 266 266 T F T <--- 3 060-079 166 164 166 274 271 274 T T T <--- 4 080-099 166 164 166 274 272 274 F T F <--- 5 100-119 166 165 166 274 273 274 T F T <--- 6 120-139 164 164 164 271 271 271 T F T <--- 7 140-159 166 164 166 274 271 274 T F T <--- 8 160-179 166 165 166 274 273 274 F F F <--- 9 180-199 166 157 166 274 262 274 F F F <--- 10 200-219 164 164 164 272 272 272 F F F 11 220-239 166 160 166 274 265 274 F T F <--- 12 240-259 161 161 161 266 266 266 T F T <--- 13 260-279 163 163 163 270 270 270 F F F 14 280-299 157 157 157 262 262 262 F F F 15 300-319 165 165 165 273 273 273 F F F 16 320-339 165 165 165 273 273 273 F F F 17 340-359 163 163 163 270 270 270 F F F 18 360-379 163 163 163 270 270 270 F F F 19 380-399 166 157 166 274 262 274 F F F <--- 20 400-419 160 160 160 265 265 265 F F F 21 420-439 166 156 166 274 261 274 F T F <--- 22 440-459 166 164 166 274 272 274 F F F <--- 23 460-479 166 164 166 274 272 274 F F F <--- 24 480-499 166 157 166 274 262 274 F F F <--- 25 500-519 162 162 162 269 269 269 T F T <--- 26 520-539 166 165 166 274 273 274 F F F <--- 27 540-559 165 165 165 273 273 273 F F F 28 560-579 164 164 164 271 271 271 F F F 29 580-599 164 164 164 272 272 272 F F F Max gen: 166, Max flush: 274 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 83 Verifying data now No change after verify Loop: 83 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 275 275 275 F F F 1 020-039 161 161 161 266 266 266 F F F 2 040-059 167 167 167 275 275 275 F F F 3 060-079 167 167 167 275 275 275 F F F 4 080-099 166 166 166 274 274 274 F F F 5 100-119 167 167 167 275 275 275 F F F 6 120-139 167 167 167 275 275 275 F F F 7 140-159 167 167 167 275 275 275 F F F 8 160-179 166 166 166 274 274 274 F F F 9 180-199 166 166 166 274 274 274 F F F 10 200-219 164 164 164 272 272 272 F F F 11 220-239 166 166 166 274 274 274 F F F 12 240-259 167 167 167 275 275 275 F F F 13 260-279 163 163 163 270 270 270 F F F 14 280-299 157 157 157 262 262 262 F F F 15 300-319 165 165 165 273 273 273 F F F 16 320-339 165 165 165 273 273 273 F F F 17 340-359 163 163 163 270 270 270 F F F 18 360-379 163 163 163 270 270 270 F F F 19 380-399 166 166 166 274 274 274 F F F 20 400-419 160 160 160 265 265 265 F F F 21 420-439 166 166 166 274 274 274 F F F 22 440-459 166 166 166 274 274 274 F F F 23 460-479 166 166 166 274 274 274 F F F 24 480-499 166 166 166 274 274 274 F F F 25 500-519 167 167 167 275 275 275 F F F 26 520-539 166 166 166 274 274 274 F F F 27 540-559 165 165 165 273 273 273 F F F 28 560-579 164 164 164 271 271 271 F F F 29 580-599 164 164 164 272 272 272 F F F Max gen: 167, Max flush: 275 Begin loop 84 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:59.911815512Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1805,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:47:59.918874358Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1805,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:59.918927996Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1805,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:59.92573328Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1805,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:59.925999632Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1805} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:59.926024632Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1805} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:59.926033091Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1805} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:59.926044461Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1805} {"msg":"Crucible 756109b0-b119-4cb0-b9c6-bcf3bd2f4592 has session id: aa78d45f-6112-4a15-b3f4-b8908f5aa88a","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:59.926360992Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1805,"session_id":"aa78d45f-6112-4a15-b3f4-b8908f5aa88a"} {"msg":"Upstairs opts: Upstairs UUID: 756109b0-b119-4cb0-b9c6-bcf3bd2f4592, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:59.926378961Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1805,"session_id":"aa78d45f-6112-4a15-b3f4-b8908f5aa88a"} {"msg":"Crucible stats registered with UUID: 756109b0-b119-4cb0-b9c6-bcf3bd2f4592","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:59.926389011Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1805,"session_id":"aa78d45f-6112-4a15-b3f4-b8908f5aa88a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:47:59.926398131Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1805,"session_id":"aa78d45f-6112-4a15-b3f4-b8908f5aa88a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:59.926535667Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1805,"":"io task","client":"1","":"downstairs","session_id":"aa78d45f-6112-4a15-b3f4-b8908f5aa88a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:59.926567886Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1805,"":"io task","client":"0","":"downstairs","session_id":"aa78d45f-6112-4a15-b3f4-b8908f5aa88a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:59.926578476Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1805,"":"io task","client":"2","":"downstairs","session_id":"aa78d45f-6112-4a15-b3f4-b8908f5aa88a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:59.926728311Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1805,"task":"crutest"} {"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:59.926742171Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1805} {"msg":"756109b0-b119-4cb0-b9c6-bcf3bd2f4592 active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:59.926758301Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1805,"session_id":"aa78d45f-6112-4a15-b3f4-b8908f5aa88a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:59.92676715Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1805,"client":"0","":"downstairs","session_id":"aa78d45f-6112-4a15-b3f4-b8908f5aa88a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:59.926777Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1805,"client":"1","":"downstairs","session_id":"aa78d45f-6112-4a15-b3f4-b8908f5aa88a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:59.92678697Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1805,"client":"2","":"downstairs","session_id":"aa78d45f-6112-4a15-b3f4-b8908f5aa88a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:59.926796519Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1805,"":"io task","client":"2","":"downstairs","session_id":"aa78d45f-6112-4a15-b3f4-b8908f5aa88a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:59.926806579Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1805,"":"io task","client":"0","":"downstairs","session_id":"aa78d45f-6112-4a15-b3f4-b8908f5aa88a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:59.926816599Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1805,"":"io task","client":"1","":"downstairs","session_id":"aa78d45f-6112-4a15-b3f4-b8908f5aa88a"} {"msg":"ds_connection connected from Ok(127.0.0.1:49055)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:59.926919446Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1805,"":"io task","client":"2","":"downstairs","session_id":"aa78d45f-6112-4a15-b3f4-b8908f5aa88a"} {"msg":"ds_connection connected from Ok(127.0.0.1:40315)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:59.926944785Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1805,"":"io task","client":"0","":"downstairs","session_id":"aa78d45f-6112-4a15-b3f4-b8908f5aa88a"} {"msg":"ds_connection connected from Ok(127.0.0.1:39069)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:59.926955915Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1805,"":"io task","client":"1","":"downstairs","session_id":"aa78d45f-6112-4a15-b3f4-b8908f5aa88a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:59.927517889Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1805,"client":"2","":"downstairs","session_id":"aa78d45f-6112-4a15-b3f4-b8908f5aa88a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:59.927555778Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1805,"client":"1","":"downstairs","session_id":"aa78d45f-6112-4a15-b3f4-b8908f5aa88a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:59.927645265Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1805,"client":"0","":"downstairs","session_id":"aa78d45f-6112-4a15-b3f4-b8908f5aa88a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:59.92782006Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1805,"session_id":"aa78d45f-6112-4a15-b3f4-b8908f5aa88a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:59.927836519Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1805,"session_id":"aa78d45f-6112-4a15-b3f4-b8908f5aa88a"} {"msg":"[0]R flush_numbers[0..12]: [275, 266, 275, 275, 274, 275, 275, 275, 274, 274, 272, 274]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:59.927935217Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1805,"":"downstairs","":"downstairs","session_id":"aa78d45f-6112-4a15-b3f4-b8908f5aa88a"} {"msg":"[0]R generation[0..12]: [167, 161, 167, 167, 166, 167, 167, 167, 166, 166, 164, 166]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:59.927946766Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1805,"":"downstairs","":"downstairs","session_id":"aa78d45f-6112-4a15-b3f4-b8908f5aa88a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:59.927957746Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1805,"":"downstairs","":"downstairs","session_id":"aa78d45f-6112-4a15-b3f4-b8908f5aa88a"} {"msg":"[1]R flush_numbers[0..12]: [275, 266, 275, 275, 274, 275, 275, 275, 274, 274, 272, 274]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:59.927968376Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1805,"":"downstairs","":"downstairs","session_id":"aa78d45f-6112-4a15-b3f4-b8908f5aa88a"} {"msg":"[1]R generation[0..12]: [167, 161, 167, 167, 166, 167, 167, 167, 166, 166, 164, 166]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:59.927978805Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1805,"":"downstairs","":"downstairs","session_id":"aa78d45f-6112-4a15-b3f4-b8908f5aa88a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:59.927989735Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1805,"":"downstairs","":"downstairs","session_id":"aa78d45f-6112-4a15-b3f4-b8908f5aa88a"} {"msg":"[2]R flush_numbers[0..12]: [275, 266, 275, 275, 274, 275, 275, 275, 274, 274, 272, 274]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:59.928000315Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1805,"":"downstairs","":"downstairs","session_id":"aa78d45f-6112-4a15-b3f4-b8908f5aa88a"} {"msg":"[2]R generation[0..12]: [167, 161, 167, 167, 166, 167, 167, 167, 166, 166, 164, 166]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:59.928010944Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1805,"":"downstairs","":"downstairs","session_id":"aa78d45f-6112-4a15-b3f4-b8908f5aa88a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:59.928021754Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1805,"":"downstairs","":"downstairs","session_id":"aa78d45f-6112-4a15-b3f4-b8908f5aa88a"} {"msg":"Max found gen is 168","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:59.928030924Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1805,"":"downstairs","":"downstairs","session_id":"aa78d45f-6112-4a15-b3f4-b8908f5aa88a"} {"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:59.928040424Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1805,"":"downstairs","":"downstairs","session_id":"aa78d45f-6112-4a15-b3f4-b8908f5aa88a"} {"msg":"Next flush: 276","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:59.928049723Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1805,"":"downstairs","":"downstairs","session_id":"aa78d45f-6112-4a15-b3f4-b8908f5aa88a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:59.928059113Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1805,"":"downstairs","":"downstairs","session_id":"aa78d45f-6112-4a15-b3f4-b8908f5aa88a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:59.928068773Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1805,"session_id":"aa78d45f-6112-4a15-b3f4-b8908f5aa88a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:59.928079092Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1805,"client":"0","":"downstairs","session_id":"aa78d45f-6112-4a15-b3f4-b8908f5aa88a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:59.928088592Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1805,"client":"1","":"downstairs","session_id":"aa78d45f-6112-4a15-b3f4-b8908f5aa88a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:59.928098382Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1805,"client":"2","":"downstairs","session_id":"aa78d45f-6112-4a15-b3f4-b8908f5aa88a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:59.928108152Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1805,"session_id":"aa78d45f-6112-4a15-b3f4-b8908f5aa88a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:59.928117271Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1805,"session_id":"aa78d45f-6112-4a15-b3f4-b8908f5aa88a"} {"msg":"756109b0-b119-4cb0-b9c6-bcf3bd2f4592 is now active with session: aa78d45f-6112-4a15-b3f4-b8908f5aa88a","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:59.928127091Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1805,"session_id":"aa78d45f-6112-4a15-b3f4-b8908f5aa88a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:59.928135711Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1805,"session_id":"aa78d45f-6112-4a15-b3f4-b8908f5aa88a"} {"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:59.928144731Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1805} {"msg":"Activated sub_volume 756109b0-b119-4cb0-b9c6-bcf3bd2f4592","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:59.92815269Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1805} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:59.92816077Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1805,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:59.928481991Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1805,"task":"crutest"} 01/30 Write block 429 len 40960 data: 15 12 11 10 7 5 8 9 6 7 02/30 Read block 429 len 28672 03/30 Write block 225 len 12288 data: 18 17 18 04/30 Read block 448 len 24576 05/30 Read block 508 len 32768 06/30 Read block 483 len 36864 07/30 Write block 368 len 28672 data: 10 10 8 6 9 9 7 08/30 Flush 09/30 Write block 7 len 32768 data: 14 15 14 14 12 9 10 10 10/30 Read block 274 len 4096 11/30 Read block 62 len 36864 12/30 Read block 211 len 32768 13/30 Read block 487 len 8192 14/30 Write block 22 len 16384 data: 12 9 9 9 15/30 Read block 170 len 20480 16/30 Read block 254 len 24576 17/30 Write block 44 len 12288 data: 5 8 8 18/30 Read block 137 len 32768 19/30 Read block 568 len 16384 20/30 Write block 181 len 40960 data: 14 14 13 15 14 15 13 11 12 12 21/30 Read block 270 len 16384 22/30 Read block 349 len 8192 23/30 Flush 24/30 Flush 25/30 Write block 7 len 16384 data: 15 16 15 15 26/30 Read block 246 len 28672 27/30 Write block 146 len 20480 data: 17 16 14 9 9 28/30 Read block 497 len 28672 29/30 Read block 269 len 12288 30/30 Read block 248 len 36864 ---------------------------------------------------------------- Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:22 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1008 Acked Write 8 Done Done Done false 1009 Acked Read 1 Done Done Done false 1010 Acked Read 9 Done Done Done false 1011 Acked Read 8 Sent Done Done false 1012 Acked Read 2 Sent Done Done false 1013 Acked Write 4 Sent Done Done false 1014 Acked Read 5 Sent Done Done false 1015 Acked Read 6 Sent Done Done false 1016 Acked Write 3 Sent Done Done false 1017 Acked Read 8 Sent Done Done false 1018 Acked Read 4 Sent Done Done false 1019 Acked Write 10 Sent Done Done false 1020 Acked Read 4 Sent Done Done false 1021 Acked Read 2 Sent Done Done false 1022 Acked Flush 0 Sent Done Done false 1023 Acked Flush 0 Sent Done Done false 1024 Acked Write 4 Sent Done Done false 1025 Acked Read 7 Sent Done Done false 1026 Acked Write 5 Sent Done Done false 1027 Acked Read 7 Sent Done Done false 1028 Acked Read 3 Sent Done Done false 1029 Acked Read 9 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 19 0 0 19 Done 3 22 22 47 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1007)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:47:59.938254029Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1805} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:47:59.938445343Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1805,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 168 168 275 277 277 T T T <--- 1 020-039 161 168 168 266 277 277 F F F <--- 2 040-059 167 168 168 275 277 277 F F F <--- 3 060-079 167 167 167 275 275 275 F F F 4 080-099 166 166 166 274 274 274 F F F 5 100-119 167 167 167 275 275 275 F F F 6 120-139 167 167 167 275 275 275 F F F 7 140-159 167 167 167 275 275 275 F T T <--- 8 160-179 166 166 166 274 274 274 F F F 9 180-199 166 168 168 274 277 277 F F F <--- 10 200-219 164 164 164 272 272 272 F F F 11 220-239 168 168 168 276 276 276 F F F 12 240-259 167 167 167 275 275 275 F F F 13 260-279 163 163 163 270 270 270 F F F 14 280-299 157 157 157 262 262 262 F F F 15 300-319 165 165 165 273 273 273 F F F 16 320-339 165 165 165 273 273 273 F F F 17 340-359 163 163 163 270 270 270 F F F 18 360-379 168 168 168 276 276 276 F F F 19 380-399 166 166 166 274 274 274 F F F 20 400-419 160 160 160 265 265 265 F F F 21 420-439 168 168 168 276 276 276 F F F 22 440-459 166 166 166 274 274 274 F F F 23 460-479 166 166 166 274 274 274 F F F 24 480-499 166 166 166 274 274 274 F F F 25 500-519 167 167 167 275 275 275 F F F 26 520-539 166 166 166 274 274 274 F F F 27 540-559 165 165 165 273 273 273 F F F 28 560-579 164 164 164 271 271 271 F F F 29 580-599 164 164 164 272 272 272 F F F Max gen: 168, Max flush: 277 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 84 Verifying data now No change after verify Loop: 84 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 169 169 169 278 278 278 F F F 1 020-039 168 168 168 277 277 277 F F F 2 040-059 168 168 168 277 277 277 F F F 3 060-079 167 167 167 275 275 275 F F F 4 080-099 166 166 166 274 274 274 F F F 5 100-119 167 167 167 275 275 275 F F F 6 120-139 167 167 167 275 275 275 F F F 7 140-159 169 169 169 278 278 278 F F F 8 160-179 166 166 166 274 274 274 F F F 9 180-199 168 168 168 277 277 277 F F F 10 200-219 164 164 164 272 272 272 F F F 11 220-239 168 168 168 276 276 276 F F F 12 240-259 167 167 167 275 275 275 F F F 13 260-279 163 163 163 270 270 270 F F F 14 280-299 157 157 157 262 262 262 F F F 15 300-319 165 165 165 273 273 273 F F F 16 320-339 165 165 165 273 273 273 F F F 17 340-359 163 163 163 270 270 270 F F F 18 360-379 168 168 168 276 276 276 F F F 19 380-399 166 166 166 274 274 274 F F F 20 400-419 160 160 160 265 265 265 F F F 21 420-439 168 168 168 276 276 276 F F F 22 440-459 166 166 166 274 274 274 F F F 23 460-479 166 166 166 274 274 274 F F F 24 480-499 166 166 166 274 274 274 F F F 25 500-519 167 167 167 275 275 275 F F F 26 520-539 166 166 166 274 274 274 F F F 27 540-559 165 165 165 273 273 273 F F F 28 560-579 164 164 164 271 271 271 F F F 29 580-599 164 164 164 272 272 272 F F F Max gen: 169, Max flush: 278 Begin loop 85 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:04.177292375Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1813,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:48:04.184299193Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1813,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:04.184349192Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1813,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:04.190981681Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1813,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:04.191244833Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1813} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:04.191273722Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1813} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:04.191282352Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1813} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:04.191295832Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1813} {"msg":"Crucible 7748021e-0e11-4071-8ed0-7a766693d350 has session id: a7a59b54-a62e-468f-b5e9-d110409e4aab","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:04.191629002Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1813,"session_id":"a7a59b54-a62e-468f-b5e9-d110409e4aab"} {"msg":"Upstairs opts: Upstairs UUID: 7748021e-0e11-4071-8ed0-7a766693d350, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:04.191657991Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1813,"session_id":"a7a59b54-a62e-468f-b5e9-d110409e4aab"} {"msg":"Crucible stats registered with UUID: 7748021e-0e11-4071-8ed0-7a766693d350","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:04.191668901Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1813,"session_id":"a7a59b54-a62e-468f-b5e9-d110409e4aab"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:48:04.191678081Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1813,"session_id":"a7a59b54-a62e-468f-b5e9-d110409e4aab"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:04.191829066Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1813,"":"io task","client":"2","":"downstairs","session_id":"a7a59b54-a62e-468f-b5e9-d110409e4aab"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:04.191861895Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1813,"":"io task","client":"0","":"downstairs","session_id":"a7a59b54-a62e-468f-b5e9-d110409e4aab"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:04.191872695Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1813,"":"io task","client":"1","":"downstairs","session_id":"a7a59b54-a62e-468f-b5e9-d110409e4aab"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:04.191972642Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1813,"task":"crutest"} {"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:04.191983392Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1813} {"msg":"7748021e-0e11-4071-8ed0-7a766693d350 active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:04.192006881Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1813,"session_id":"a7a59b54-a62e-468f-b5e9-d110409e4aab"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:04.192016241Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1813,"client":"0","":"downstairs","session_id":"a7a59b54-a62e-468f-b5e9-d110409e4aab"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:04.19202594Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1813,"client":"1","":"downstairs","session_id":"a7a59b54-a62e-468f-b5e9-d110409e4aab"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:04.19203558Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1813,"client":"2","":"downstairs","session_id":"a7a59b54-a62e-468f-b5e9-d110409e4aab"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:04.19204517Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1813,"":"io task","client":"2","":"downstairs","session_id":"a7a59b54-a62e-468f-b5e9-d110409e4aab"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:04.19205717Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1813,"":"io task","client":"0","":"downstairs","session_id":"a7a59b54-a62e-468f-b5e9-d110409e4aab"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:04.192067059Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1813,"":"io task","client":"1","":"downstairs","session_id":"a7a59b54-a62e-468f-b5e9-d110409e4aab"} {"msg":"ds_connection connected from Ok(127.0.0.1:41804)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:04.192173686Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1813,"":"io task","client":"0","":"downstairs","session_id":"a7a59b54-a62e-468f-b5e9-d110409e4aab"} {"msg":"ds_connection connected from Ok(127.0.0.1:38590)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:04.192184696Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1813,"":"io task","client":"2","":"downstairs","session_id":"a7a59b54-a62e-468f-b5e9-d110409e4aab"} {"msg":"ds_connection connected from Ok(127.0.0.1:45169)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:04.192194856Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1813,"":"io task","client":"1","":"downstairs","session_id":"a7a59b54-a62e-468f-b5e9-d110409e4aab"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:04.192903935Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1813,"client":"1","":"downstairs","session_id":"a7a59b54-a62e-468f-b5e9-d110409e4aab"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:04.192933604Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1813,"client":"2","":"downstairs","session_id":"a7a59b54-a62e-468f-b5e9-d110409e4aab"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:04.192945044Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1813,"client":"0","":"downstairs","session_id":"a7a59b54-a62e-468f-b5e9-d110409e4aab"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:04.193264645Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1813,"session_id":"a7a59b54-a62e-468f-b5e9-d110409e4aab"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:04.193274634Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1813,"session_id":"a7a59b54-a62e-468f-b5e9-d110409e4aab"} {"msg":"[0]R flush_numbers[0..12]: [278, 277, 277, 275, 274, 275, 275, 278, 274, 277, 272, 276]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:04.193293224Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1813,"":"downstairs","":"downstairs","session_id":"a7a59b54-a62e-468f-b5e9-d110409e4aab"} {"msg":"[0]R generation[0..12]: [169, 168, 168, 167, 166, 167, 167, 169, 166, 168, 164, 168]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:04.193304094Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1813,"":"downstairs","":"downstairs","session_id":"a7a59b54-a62e-468f-b5e9-d110409e4aab"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:04.193315053Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1813,"":"downstairs","":"downstairs","session_id":"a7a59b54-a62e-468f-b5e9-d110409e4aab"} {"msg":"[1]R flush_numbers[0..12]: [278, 277, 277, 275, 274, 275, 275, 278, 274, 277, 272, 276]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:04.193325683Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1813,"":"downstairs","":"downstairs","session_id":"a7a59b54-a62e-468f-b5e9-d110409e4aab"} {"msg":"[1]R generation[0..12]: [169, 168, 168, 167, 166, 167, 167, 169, 166, 168, 164, 168]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:04.193336253Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1813,"":"downstairs","":"downstairs","session_id":"a7a59b54-a62e-468f-b5e9-d110409e4aab"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:04.193349422Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1813,"":"downstairs","":"downstairs","session_id":"a7a59b54-a62e-468f-b5e9-d110409e4aab"} {"msg":"[2]R flush_numbers[0..12]: [278, 277, 277, 275, 274, 275, 275, 278, 274, 277, 272, 276]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:04.193359902Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1813,"":"downstairs","":"downstairs","session_id":"a7a59b54-a62e-468f-b5e9-d110409e4aab"} {"msg":"[2]R generation[0..12]: [169, 168, 168, 167, 166, 167, 167, 169, 166, 168, 164, 168]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:04.193370502Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1813,"":"downstairs","":"downstairs","session_id":"a7a59b54-a62e-468f-b5e9-d110409e4aab"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:04.193381191Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1813,"":"downstairs","":"downstairs","session_id":"a7a59b54-a62e-468f-b5e9-d110409e4aab"} {"msg":"Max found gen is 170","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:04.193390551Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1813,"":"downstairs","":"downstairs","session_id":"a7a59b54-a62e-468f-b5e9-d110409e4aab"} {"msg":"Generation requested: 170 >= found:170","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:04.193400261Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1813,"":"downstairs","":"downstairs","session_id":"a7a59b54-a62e-468f-b5e9-d110409e4aab"} {"msg":"Next flush: 279","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:04.193409441Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1813,"":"downstairs","":"downstairs","session_id":"a7a59b54-a62e-468f-b5e9-d110409e4aab"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:04.19341872Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1813,"":"downstairs","":"downstairs","session_id":"a7a59b54-a62e-468f-b5e9-d110409e4aab"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:04.19342842Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1813,"session_id":"a7a59b54-a62e-468f-b5e9-d110409e4aab"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:04.19343704Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1813,"client":"0","":"downstairs","session_id":"a7a59b54-a62e-468f-b5e9-d110409e4aab"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:04.193446569Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1813,"client":"1","":"downstairs","session_id":"a7a59b54-a62e-468f-b5e9-d110409e4aab"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:04.193456129Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1813,"client":"2","":"downstairs","session_id":"a7a59b54-a62e-468f-b5e9-d110409e4aab"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:04.193466199Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1813,"session_id":"a7a59b54-a62e-468f-b5e9-d110409e4aab"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:04.193475349Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1813,"session_id":"a7a59b54-a62e-468f-b5e9-d110409e4aab"} {"msg":"7748021e-0e11-4071-8ed0-7a766693d350 is now active with session: a7a59b54-a62e-468f-b5e9-d110409e4aab","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:04.193485278Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1813,"session_id":"a7a59b54-a62e-468f-b5e9-d110409e4aab"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:04.193495058Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1813,"session_id":"a7a59b54-a62e-468f-b5e9-d110409e4aab"} {"msg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:04.193504238Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1813} {"msg":"Activated sub_volume 7748021e-0e11-4071-8ed0-7a766693d350","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:04.193512288Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1813} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:04.193520297Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1813,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:04.193818559Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1813,"task":"crutest"} 01/30 Write block 18 len 16384 data: 11 8 8 7 02/30 Write block 26 len 4096 data: 6 03/30 Read block 192 len 20480 04/30 Flush 05/30 Write block 390 len 4096 data: 12 06/30 Write block 226 len 24576 data: 18 19 16 16 18 19 07/30 Flush 08/30 Write block 289 len 20480 data: 13 12 12 13 16 09/30 Read block 404 len 28672 10/30 Flush 11/30 Write block 513 len 24576 data: 11 11 9 9 10 12 12/30 Read block 107 len 12288 13/30 Read block 370 len 32768 14/30 Write block 522 len 36864 data: 16 15 14 11 10 10 7 10 10 15/30 Read block 213 len 24576 16/30 Read block 273 len 12288 17/30 Write block 422 len 32768 data: 11 12 13 13 13 13 12 16 18/30 Write block 399 len 8192 data: 14 15 19/30 Flush 20/30 Write block 500 len 28672 data: 14 18 18 20 21 19 16 21/30 Read block 80 len 8192 22/30 Write block 342 len 36864 data: 9 10 10 11 12 11 11 10 8 23/30 Read block 366 len 24576 24/30 Write block 398 len 12288 data: 10 15 16 25/30 Read block 472 len 40960 26/30 Write block 165 len 32768 data: 10 13 12 11 12 13 13 16 27/30 Write block 67 len 28672 data: 8 6 8 12 12 13 14 28/30 Read block 308 len 20480 29/30 Write block 236 len 20480 data: 10 12 10 11 12 30/30 Read block 122 len 12288 ---------------------------------------------------------------- Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Write 1 Sent Done Done false 1002 Acked Read 5 Sent Done Done false 1003 Acked Flush 0 Sent Done Done false 1004 Acked Write 1 Sent Done Done false 1005 Acked Write 6 Sent Done Done false 1006 Acked Flush 0 Sent Done Done false 1007 Acked Write 5 Sent Done Done false 1008 Acked Read 7 Sent Done Done false 1009 Acked Flush 0 Sent Done Done false 1010 Acked Write 6 Sent Done Done false 1011 Acked Read 3 Sent Done Done false 1012 Acked Read 8 Sent Done Done false 1013 Acked Write 9 Sent Done Done false 1014 Acked Read 6 Sent Done Done false 1015 Acked Read 3 Sent Done Done false 1016 Acked Write 8 Sent Done Done false 1017 Acked Write 2 Sent Done Done false 1018 Acked Flush 0 Sent Done Done false 1019 Acked Write 7 Sent Done Done false 1020 Acked Read 2 Sent Done Done false 1021 Acked Write 9 Sent Done Done false 1022 Acked Read 6 Sent Done Done false 1023 Acked Write 3 Sent Done Done false 1024 Acked Read 10 Sent Done Done false 1025 Acked Write 8 Sent Done Done false 1026 Acked Write 7 Sent Done Done false 1027 Acked Read 5 Sent Done Done false 1028 Acked Write 5 Sent Done Done false 1029 Acked Read 3 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 0 1 30 Done 1 30 29 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:48:04.203024973Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1813} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:04.203232557Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1813,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 169 170 170 278 279 279 T F F <--- 1 020-039 168 170 170 277 279 279 T F F <--- 2 040-059 168 168 168 277 277 277 F F F 3 060-079 167 167 167 275 275 275 F T T <--- 4 080-099 166 166 166 274 274 274 F F F 5 100-119 167 167 167 275 275 275 F F F 6 120-139 167 167 167 275 275 275 F F F 7 140-159 169 169 169 278 278 278 F F F 8 160-179 166 166 166 274 274 274 F T T <--- 9 180-199 168 168 168 277 277 277 F F F 10 200-219 164 164 164 272 272 272 F F F 11 220-239 168 170 170 276 280 280 F T T <--- 12 240-259 167 167 167 275 275 275 F T T <--- 13 260-279 163 163 163 270 270 270 F F F 14 280-299 157 170 170 262 281 281 F F F <--- 15 300-319 165 165 165 273 273 273 F F F 16 320-339 165 165 165 273 273 273 F F F 17 340-359 163 163 163 270 270 270 F T T <--- 18 360-379 168 168 168 276 276 276 F F F 19 380-399 166 170 170 274 282 282 F T T <--- 20 400-419 160 170 170 265 282 282 F T T <--- 21 420-439 168 170 170 276 282 282 F F F <--- 22 440-459 166 166 166 274 274 274 F F F 23 460-479 166 166 166 274 274 274 F F F 24 480-499 166 166 166 274 274 274 F F F 25 500-519 167 170 170 275 282 282 F T T <--- 26 520-539 166 170 170 274 282 282 F F F <--- 27 540-559 165 165 165 273 273 273 F F F 28 560-579 164 164 164 271 271 271 F F F 29 580-599 164 164 164 272 272 272 F F F Max gen: 170, Max flush: 282 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 85 Verifying data now No change after verify Loop: 85 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 170 170 170 279 279 279 F F F 1 020-039 170 170 170 279 279 279 F F F 2 040-059 168 168 168 277 277 277 F F F 3 060-079 171 171 171 283 283 283 F F F 4 080-099 166 166 166 274 274 274 F F F 5 100-119 167 167 167 275 275 275 F F F 6 120-139 167 167 167 275 275 275 F F F 7 140-159 169 169 169 278 278 278 F F F 8 160-179 171 171 171 283 283 283 F F F 9 180-199 168 168 168 277 277 277 F F F 10 200-219 164 164 164 272 272 272 F F F 11 220-239 171 171 171 283 283 283 F F F 12 240-259 171 171 171 283 283 283 F F F 13 260-279 163 163 163 270 270 270 F F F 14 280-299 170 170 170 281 281 281 F F F 15 300-319 165 165 165 273 273 273 F F F 16 320-339 165 165 165 273 273 273 F F F 17 340-359 171 171 171 283 283 283 F F F 18 360-379 168 168 168 276 276 276 F F F 19 380-399 171 171 171 283 283 283 F F F 20 400-419 171 171 171 283 283 283 F F F 21 420-439 170 170 170 282 282 282 F F F 22 440-459 166 166 166 274 274 274 F F F 23 460-479 166 166 166 274 274 274 F F F 24 480-499 166 166 166 274 274 274 F F F 25 500-519 171 171 171 283 283 283 F F F 26 520-539 170 170 170 282 282 282 F F F 27 540-559 165 165 165 273 273 273 F F F 28 560-579 164 164 164 271 271 271 F F F 29 580-599 164 164 164 272 272 272 F F F Max gen: 171, Max flush: 283 Begin loop 86 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:08.462353524Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1821,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:08.470530649Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1821,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:08.470800391Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1821} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:08.47082745Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1821} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:08.47083637Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1821} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:08.470847979Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1821} {"msg":"Crucible 24f79e26-8133-4e41-a6cb-bf1554ad873c has session id: 672bd732-a3a5-4bcb-bfa4-6d7a1d67d774","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:08.471152881Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1821,"session_id":"672bd732-a3a5-4bcb-bfa4-6d7a1d67d774"} {"msg":"Upstairs opts: Upstairs UUID: 24f79e26-8133-4e41-a6cb-bf1554ad873c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:08.47119261Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1821,"session_id":"672bd732-a3a5-4bcb-bfa4-6d7a1d67d774"} {"msg":"Crucible stats registered with UUID: 24f79e26-8133-4e41-a6cb-bf1554ad873c","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:08.471203169Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1821,"session_id":"672bd732-a3a5-4bcb-bfa4-6d7a1d67d774"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:48:08.471214489Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1821,"session_id":"672bd732-a3a5-4bcb-bfa4-6d7a1d67d774"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:08.471343325Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1821,"":"io task","client":"0","":"downstairs","session_id":"672bd732-a3a5-4bcb-bfa4-6d7a1d67d774"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:08.471376074Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1821,"":"io task","client":"1","":"downstairs","session_id":"672bd732-a3a5-4bcb-bfa4-6d7a1d67d774"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:08.471387074Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1821,"":"io task","client":"2","":"downstairs","session_id":"672bd732-a3a5-4bcb-bfa4-6d7a1d67d774"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:08.47151781Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1821,"task":"crutest"} {"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:08.47153203Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1821} {"msg":"24f79e26-8133-4e41-a6cb-bf1554ad873c active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:08.471543079Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1821,"session_id":"672bd732-a3a5-4bcb-bfa4-6d7a1d67d774"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:08.471551929Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1821,"client":"0","":"downstairs","session_id":"672bd732-a3a5-4bcb-bfa4-6d7a1d67d774"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:08.471561599Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1821,"client":"1","":"downstairs","session_id":"672bd732-a3a5-4bcb-bfa4-6d7a1d67d774"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:08.471571429Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1821,"client":"2","":"downstairs","session_id":"672bd732-a3a5-4bcb-bfa4-6d7a1d67d774"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:08.471581068Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1821,"":"io task","client":"2","":"downstairs","session_id":"672bd732-a3a5-4bcb-bfa4-6d7a1d67d774"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:08.471590958Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1821,"":"io task","client":"0","":"downstairs","session_id":"672bd732-a3a5-4bcb-bfa4-6d7a1d67d774"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:08.471600918Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1821,"":"io task","client":"1","":"downstairs","session_id":"672bd732-a3a5-4bcb-bfa4-6d7a1d67d774"} {"msg":"ds_connection connected from Ok(127.0.0.1:43881)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:08.471655146Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1821,"":"io task","client":"0","":"downstairs","session_id":"672bd732-a3a5-4bcb-bfa4-6d7a1d67d774"} {"msg":"ds_connection connected from Ok(127.0.0.1:38012)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:08.471673566Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1821,"":"io task","client":"2","":"downstairs","session_id":"672bd732-a3a5-4bcb-bfa4-6d7a1d67d774"} {"msg":"ds_connection connected from Ok(127.0.0.1:45475)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:08.471684005Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1821,"":"io task","client":"1","":"downstairs","session_id":"672bd732-a3a5-4bcb-bfa4-6d7a1d67d774"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:08.472329707Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1821,"client":"2","":"downstairs","session_id":"672bd732-a3a5-4bcb-bfa4-6d7a1d67d774"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:08.472355036Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1821,"client":"0","":"downstairs","session_id":"672bd732-a3a5-4bcb-bfa4-6d7a1d67d774"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:08.472366666Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1821,"client":"1","":"downstairs","session_id":"672bd732-a3a5-4bcb-bfa4-6d7a1d67d774"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:08.472650117Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1821,"session_id":"672bd732-a3a5-4bcb-bfa4-6d7a1d67d774"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:08.472687966Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1821,"session_id":"672bd732-a3a5-4bcb-bfa4-6d7a1d67d774"} {"msg":"[0]R flush_numbers[0..12]: [279, 279, 277, 283, 274, 275, 275, 278, 283, 277, 272, 283]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:08.472751465Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1821,"":"downstairs","":"downstairs","session_id":"672bd732-a3a5-4bcb-bfa4-6d7a1d67d774"} {"msg":"[0]R generation[0..12]: [170, 170, 168, 171, 166, 167, 167, 169, 171, 168, 164, 171]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:08.472762754Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1821,"":"downstairs","":"downstairs","session_id":"672bd732-a3a5-4bcb-bfa4-6d7a1d67d774"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:08.472773774Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1821,"":"downstairs","":"downstairs","session_id":"672bd732-a3a5-4bcb-bfa4-6d7a1d67d774"} {"msg":"[1]R flush_numbers[0..12]: [279, 279, 277, 283, 274, 275, 275, 278, 283, 277, 272, 283]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:08.472784354Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1821,"":"downstairs","":"downstairs","session_id":"672bd732-a3a5-4bcb-bfa4-6d7a1d67d774"} {"msg":"[1]R generation[0..12]: [170, 170, 168, 171, 166, 167, 167, 169, 171, 168, 164, 171]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:08.472794963Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1821,"":"downstairs","":"downstairs","session_id":"672bd732-a3a5-4bcb-bfa4-6d7a1d67d774"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:08.472805833Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1821,"":"downstairs","":"downstairs","session_id":"672bd732-a3a5-4bcb-bfa4-6d7a1d67d774"} {"msg":"[2]R flush_numbers[0..12]: [279, 279, 277, 283, 274, 275, 275, 278, 283, 277, 272, 283]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:08.472816443Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1821,"":"downstairs","":"downstairs","session_id":"672bd732-a3a5-4bcb-bfa4-6d7a1d67d774"} {"msg":"[2]R generation[0..12]: [170, 170, 168, 171, 166, 167, 167, 169, 171, 168, 164, 171]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:08.472826942Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1821,"":"downstairs","":"downstairs","session_id":"672bd732-a3a5-4bcb-bfa4-6d7a1d67d774"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:08.472837712Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1821,"":"downstairs","":"downstairs","session_id":"672bd732-a3a5-4bcb-bfa4-6d7a1d67d774"} {"msg":"Max found gen is 172","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:08.472849782Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1821,"":"downstairs","":"downstairs","session_id":"672bd732-a3a5-4bcb-bfa4-6d7a1d67d774"} {"msg":"Generation requested: 172 >= found:172","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:08.472859401Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1821,"":"downstairs","":"downstairs","session_id":"672bd732-a3a5-4bcb-bfa4-6d7a1d67d774"} {"msg":"Next flush: 284","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:08.472868681Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1821,"":"downstairs","":"downstairs","session_id":"672bd732-a3a5-4bcb-bfa4-6d7a1d67d774"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:08.472878081Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1821,"":"downstairs","":"downstairs","session_id":"672bd732-a3a5-4bcb-bfa4-6d7a1d67d774"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:08.472888161Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1821,"session_id":"672bd732-a3a5-4bcb-bfa4-6d7a1d67d774"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:08.47289701Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1821,"client":"0","":"downstairs","session_id":"672bd732-a3a5-4bcb-bfa4-6d7a1d67d774"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:08.47290688Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1821,"client":"1","":"downstairs","session_id":"672bd732-a3a5-4bcb-bfa4-6d7a1d67d774"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:08.47291642Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1821,"client":"2","":"downstairs","session_id":"672bd732-a3a5-4bcb-bfa4-6d7a1d67d774"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:08.47292608Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1821,"session_id":"672bd732-a3a5-4bcb-bfa4-6d7a1d67d774"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:08.472935209Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1821,"session_id":"672bd732-a3a5-4bcb-bfa4-6d7a1d67d774"} {"msg":"24f79e26-8133-4e41-a6cb-bf1554ad873c is now active with session: 672bd732-a3a5-4bcb-bfa4-6d7a1d67d774","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:08.472945109Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1821,"session_id":"672bd732-a3a5-4bcb-bfa4-6d7a1d67d774"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:08.472953859Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1821,"session_id":"672bd732-a3a5-4bcb-bfa4-6d7a1d67d774"} {"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:08.472963008Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1821} {"msg":"Activated sub_volume 24f79e26-8133-4e41-a6cb-bf1554ad873c","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:08.472971128Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1821} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:08.472979218Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1821,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:08.47326173Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1821,"task":"crutest"} 01/30 Write block 366 len 32768 data: 11 13 11 11 9 7 10 10 02/30 Flush 03/30 Write block 409 len 24576 data: 10 13 16 15 15 13 04/30 Write block 60 len 16384 data: 11 13 13 10 05/30 Write block 422 len 4096 data: 12 06/30 Read block 572 len 36864 07/30 Write block 363 len 8192 data: 18 18 08/30 Read block 375 len 16384 09/30 Read block 524 len 24576 10/30 Flush 11/30 Write block 375 len 16384 data: 12 17 18 19 12/30 Read block 121 len 16384 13/30 Write block 56 len 28672 data: 14 13 15 13 12 14 14 14/30 Read block 329 len 32768 15/30 Write block 91 len 40960 data: 9 9 11 13 15 14 15 13 12 12 16/30 Read block 383 len 12288 17/30 Write block 321 len 20480 data: 14 13 15 20 20 18/30 Read block 428 len 28672 19/30 Write block 569 len 28672 data: 10 10 12 11 12 12 13 20/30 Write block 420 len 4096 data: 14 21/30 Write block 4 len 16384 data: 11 11 14 16 22/30 Read block 6 len 32768 23/30 Flush 24/30 Write block 305 len 40960 data: 11 10 10 8 11 13 13 13 15 15 25/30 Flush 26/30 Write block 388 len 32768 data: 13 11 13 12 13 14 14 16 27/30 Write block 242 len 4096 data: 15 28/30 Write block 319 len 16384 data: 11 10 15 14 29/30 Read block 219 len 32768 30/30 Write block 513 len 36864 data: 12 12 10 10 11 13 13 12 15 ---------------------------------------------------------------- Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Sent Done false 1001 Acked Flush 0 Done Sent Done false 1002 Acked Write 6 Done Sent Done false 1003 Acked Write 4 Done Sent Done false 1004 Acked Write 1 Done Sent Done false 1005 Acked Read 9 Done Sent Done false 1006 Acked Write 2 Done Sent Done false 1007 Acked Read 4 Done Sent Done false 1008 Acked Read 6 Done Sent Done false 1009 Acked Flush 0 Done Sent Done false 1010 Acked Write 4 Done Sent Done false 1011 Acked Read 4 Done Sent Done false 1012 Acked Write 7 Done Sent Done false 1013 Acked Read 8 Done Sent Done false 1014 Acked Write 10 Done Sent Done false 1015 Acked Read 3 Done Sent Done false 1016 Acked Write 5 Done Sent Done false 1017 Acked Read 7 Done Sent Done false 1018 Acked Write 7 Done Sent Done false 1019 Acked Write 1 Done Sent Done false 1020 Acked Write 4 Done Sent Done false 1021 Acked Read 8 Done Sent Done false 1022 Acked Flush 0 Done Sent Done false 1023 Acked Write 10 Done Sent Done false 1024 Acked Flush 0 Done Sent Done false 1025 Acked Write 8 Done Sent Done false 1026 Acked Write 1 Done Sent Done false 1027 Acked Write 4 Done Sent Done false 1028 Acked Read 8 Done Sent Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 30 1 32 Done 29 0 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) None Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:48:08.482150623Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1821} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:08.482353038Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1821,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 170 172 286 279 286 F F F <--- 1 020-039 170 170 170 279 279 279 F F F 2 040-059 172 168 172 286 277 286 F F F <--- 3 060-079 172 171 172 286 283 286 F F F <--- 4 080-099 172 166 172 286 274 286 F F F <--- 5 100-119 172 167 172 286 275 286 F F F <--- 6 120-139 167 167 167 275 275 275 F F F 7 140-159 169 169 169 278 278 278 F F F 8 160-179 171 171 171 283 283 283 F F F 9 180-199 168 168 168 277 277 277 F F F 10 200-219 164 164 164 272 272 272 F F F 11 220-239 171 171 171 283 283 283 F F F 12 240-259 171 171 171 283 283 283 T F T <--- 13 260-279 163 163 163 270 270 270 F F F 14 280-299 170 170 170 281 281 281 F F F 15 300-319 172 165 172 287 273 287 T F T <--- 16 320-339 172 165 172 286 273 286 T F T <--- 17 340-359 171 171 171 283 283 283 F F F 18 360-379 172 168 172 286 276 286 F F F <--- 19 380-399 171 171 171 283 283 283 T F T <--- 20 400-419 172 171 172 285 283 285 F F F <--- 21 420-439 172 170 172 286 282 286 F F F <--- 22 440-459 166 166 166 274 274 274 F F F 23 460-479 166 166 166 274 274 274 F F F 24 480-499 166 166 166 274 274 274 F F F 25 500-519 171 171 171 283 283 283 T F T <--- 26 520-539 170 170 170 282 282 282 T F T <--- 27 540-559 165 165 165 273 273 273 F F F 28 560-579 172 164 172 286 271 286 F F F <--- 29 580-599 164 164 164 272 272 272 F F F Max gen: 172, Max flush: 287 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 86 Verifying data now No change after verify Loop: 86 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 172 172 286 286 286 F F F 1 020-039 170 170 170 279 279 279 F F F 2 040-059 172 172 172 286 286 286 F F F 3 060-079 172 172 172 286 286 286 F F F 4 080-099 172 172 172 286 286 286 F F F 5 100-119 172 172 172 286 286 286 F F F 6 120-139 167 167 167 275 275 275 F F F 7 140-159 169 169 169 278 278 278 F F F 8 160-179 171 171 171 283 283 283 F F F 9 180-199 168 168 168 277 277 277 F F F 10 200-219 164 164 164 272 272 272 F F F 11 220-239 171 171 171 283 283 283 F F F 12 240-259 173 173 173 288 288 288 F F F 13 260-279 163 163 163 270 270 270 F F F 14 280-299 170 170 170 281 281 281 F F F 15 300-319 173 173 173 288 288 288 F F F 16 320-339 173 173 173 288 288 288 F F F 17 340-359 171 171 171 283 283 283 F F F 18 360-379 172 172 172 286 286 286 F F F 19 380-399 173 173 173 288 288 288 F F F 20 400-419 172 172 172 285 285 285 F F F 21 420-439 172 172 172 286 286 286 F F F 22 440-459 166 166 166 274 274 274 F F F 23 460-479 166 166 166 274 274 274 F F F 24 480-499 166 166 166 274 274 274 F F F 25 500-519 173 173 173 288 288 288 F F F 26 520-539 173 173 173 288 288 288 F F F 27 540-559 165 165 165 273 273 273 F F F 28 560-579 172 172 172 286 286 286 F F F 29 580-599 164 164 164 272 272 272 F F F Max gen: 173, Max flush: 288 Begin loop 87 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:12.733240352Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1829,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:48:12.74026085Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1829,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:12.740314568Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1829,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:12.747157161Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1829,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:12.747424223Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1829} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:12.747458952Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1829} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:12.747468052Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1829} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:12.747479712Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1829} {"msg":"Crucible 727a420d-1a49-40a8-991b-e0d22a6712c7 has session id: 1d847d87-d67e-4422-8d1c-80d216e62b14","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:12.747806152Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1829,"session_id":"1d847d87-d67e-4422-8d1c-80d216e62b14"} {"msg":"Upstairs opts: Upstairs UUID: 727a420d-1a49-40a8-991b-e0d22a6712c7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:12.747827202Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1829,"session_id":"1d847d87-d67e-4422-8d1c-80d216e62b14"} {"msg":"Crucible stats registered with UUID: 727a420d-1a49-40a8-991b-e0d22a6712c7","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:12.747841761Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1829,"session_id":"1d847d87-d67e-4422-8d1c-80d216e62b14"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:48:12.747850781Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1829,"session_id":"1d847d87-d67e-4422-8d1c-80d216e62b14"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:12.747986767Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1829,"":"io task","client":"2","":"downstairs","session_id":"1d847d87-d67e-4422-8d1c-80d216e62b14"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:12.748004686Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1829,"":"io task","client":"0","":"downstairs","session_id":"1d847d87-d67e-4422-8d1c-80d216e62b14"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:12.748017026Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1829,"":"io task","client":"1","":"downstairs","session_id":"1d847d87-d67e-4422-8d1c-80d216e62b14"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:12.748172352Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1829,"task":"crutest"} {"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:12.748187901Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1829} {"msg":"727a420d-1a49-40a8-991b-e0d22a6712c7 active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:12.748203551Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1829,"session_id":"1d847d87-d67e-4422-8d1c-80d216e62b14"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:12.74821236Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1829,"client":"0","":"downstairs","session_id":"1d847d87-d67e-4422-8d1c-80d216e62b14"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:12.74822217Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1829,"client":"1","":"downstairs","session_id":"1d847d87-d67e-4422-8d1c-80d216e62b14"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:12.74823172Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1829,"client":"2","":"downstairs","session_id":"1d847d87-d67e-4422-8d1c-80d216e62b14"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:12.74824114Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1829,"":"io task","client":"2","":"downstairs","session_id":"1d847d87-d67e-4422-8d1c-80d216e62b14"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:12.748251069Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1829,"":"io task","client":"0","":"downstairs","session_id":"1d847d87-d67e-4422-8d1c-80d216e62b14"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:12.748261079Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1829,"":"io task","client":"1","":"downstairs","session_id":"1d847d87-d67e-4422-8d1c-80d216e62b14"} {"msg":"ds_connection connected from Ok(127.0.0.1:33818)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:12.748384225Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1829,"":"io task","client":"0","":"downstairs","session_id":"1d847d87-d67e-4422-8d1c-80d216e62b14"} {"msg":"ds_connection connected from Ok(127.0.0.1:39175)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:12.748525541Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1829,"":"io task","client":"1","":"downstairs","session_id":"1d847d87-d67e-4422-8d1c-80d216e62b14"} {"msg":"ds_connection connected from Ok(127.0.0.1:62897)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:12.748549681Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1829,"":"io task","client":"2","":"downstairs","session_id":"1d847d87-d67e-4422-8d1c-80d216e62b14"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:12.748998758Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1829,"client":"2","":"downstairs","session_id":"1d847d87-d67e-4422-8d1c-80d216e62b14"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:12.749025227Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1829,"client":"1","":"downstairs","session_id":"1d847d87-d67e-4422-8d1c-80d216e62b14"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:12.749141984Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1829,"client":"0","":"downstairs","session_id":"1d847d87-d67e-4422-8d1c-80d216e62b14"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:12.749293449Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1829,"session_id":"1d847d87-d67e-4422-8d1c-80d216e62b14"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:12.749309119Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1829,"session_id":"1d847d87-d67e-4422-8d1c-80d216e62b14"} {"msg":"[0]R flush_numbers[0..12]: [286, 279, 286, 286, 286, 286, 275, 278, 283, 277, 272, 283]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:12.749409826Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1829,"":"downstairs","":"downstairs","session_id":"1d847d87-d67e-4422-8d1c-80d216e62b14"} {"msg":"[0]R generation[0..12]: [172, 170, 172, 172, 172, 172, 167, 169, 171, 168, 164, 171]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:12.749420946Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1829,"":"downstairs","":"downstairs","session_id":"1d847d87-d67e-4422-8d1c-80d216e62b14"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:12.749431845Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1829,"":"downstairs","":"downstairs","session_id":"1d847d87-d67e-4422-8d1c-80d216e62b14"} {"msg":"[1]R flush_numbers[0..12]: [286, 279, 286, 286, 286, 286, 275, 278, 283, 277, 272, 283]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:12.749442375Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1829,"":"downstairs","":"downstairs","session_id":"1d847d87-d67e-4422-8d1c-80d216e62b14"} {"msg":"[1]R generation[0..12]: [172, 170, 172, 172, 172, 172, 167, 169, 171, 168, 164, 171]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:12.749452865Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1829,"":"downstairs","":"downstairs","session_id":"1d847d87-d67e-4422-8d1c-80d216e62b14"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:12.749463684Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1829,"":"downstairs","":"downstairs","session_id":"1d847d87-d67e-4422-8d1c-80d216e62b14"} {"msg":"[2]R flush_numbers[0..12]: [286, 279, 286, 286, 286, 286, 275, 278, 283, 277, 272, 283]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:12.749474244Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1829,"":"downstairs","":"downstairs","session_id":"1d847d87-d67e-4422-8d1c-80d216e62b14"} {"msg":"[2]R generation[0..12]: [172, 170, 172, 172, 172, 172, 167, 169, 171, 168, 164, 171]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:12.749484684Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1829,"":"downstairs","":"downstairs","session_id":"1d847d87-d67e-4422-8d1c-80d216e62b14"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:12.749495473Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1829,"":"downstairs","":"downstairs","session_id":"1d847d87-d67e-4422-8d1c-80d216e62b14"} {"msg":"Max found gen is 174","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:12.749504723Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1829,"":"downstairs","":"downstairs","session_id":"1d847d87-d67e-4422-8d1c-80d216e62b14"} {"msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:12.749514283Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1829,"":"downstairs","":"downstairs","session_id":"1d847d87-d67e-4422-8d1c-80d216e62b14"} {"msg":"Next flush: 289","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:12.749523493Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1829,"":"downstairs","":"downstairs","session_id":"1d847d87-d67e-4422-8d1c-80d216e62b14"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:12.749534332Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1829,"":"downstairs","":"downstairs","session_id":"1d847d87-d67e-4422-8d1c-80d216e62b14"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:12.749544012Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1829,"session_id":"1d847d87-d67e-4422-8d1c-80d216e62b14"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:12.749552522Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1829,"client":"0","":"downstairs","session_id":"1d847d87-d67e-4422-8d1c-80d216e62b14"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:12.749562082Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1829,"client":"1","":"downstairs","session_id":"1d847d87-d67e-4422-8d1c-80d216e62b14"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:12.749571601Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1829,"client":"2","":"downstairs","session_id":"1d847d87-d67e-4422-8d1c-80d216e62b14"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:12.749581371Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1829,"session_id":"1d847d87-d67e-4422-8d1c-80d216e62b14"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:12.749590381Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1829,"session_id":"1d847d87-d67e-4422-8d1c-80d216e62b14"} {"msg":"727a420d-1a49-40a8-991b-e0d22a6712c7 is now active with session: 1d847d87-d67e-4422-8d1c-80d216e62b14","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:12.74960015Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1829,"session_id":"1d847d87-d67e-4422-8d1c-80d216e62b14"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:12.74960894Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1829,"session_id":"1d847d87-d67e-4422-8d1c-80d216e62b14"} {"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:12.74961792Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1829} {"msg":"Activated sub_volume 727a420d-1a49-40a8-991b-e0d22a6712c7","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:12.74962585Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1829} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:12.749633989Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1829,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:12.74995536Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1829,"task":"crutest"} 01/30 Write block 544 len 32768 data: 17 15 12 11 13 16 14 9 02/30 Write block 390 len 8192 data: 14 13 03/30 Write block 139 len 16384 data: 12 14 17 18 04/30 Flush 05/30 Write block 238 len 40960 data: 11 12 13 16 16 14 14 14 16 17 06/30 Read block 121 len 24576 07/30 Read block 3 len 40960 08/30 Read block 347 len 40960 09/30 Read block 217 len 32768 10/30 Write block 251 len 32768 data: 11 9 11 10 15 15 12 13 11/30 Flush 12/30 Write block 278 len 24576 data: 12 12 13 13 9 10 13/30 Write block 323 len 40960 data: 16 21 21 21 22 17 16 14 15 12 14/30 Write block 122 len 24576 data: 14 16 14 14 13 13 15/30 Write block 580 len 8192 data: 12 12 16/30 Read block 528 len 32768 17/30 Write block 513 len 12288 data: 13 13 11 18/30 Read block 516 len 20480 19/30 Read block 179 len 28672 20/30 Read block 375 len 20480 21/30 Read block 180 len 28672 22/30 Read block 415 len 12288 23/30 Flush 24/30 Write block 158 len 4096 data: 16 25/30 Write block 223 len 8192 data: 15 16 26/30 Write block 507 len 24576 data: 19 13 12 13 19 15 27/30 Write block 155 len 20480 data: 10 13 12 17 17 28/30 Write block 81 len 36864 data: 11 12 12 12 11 11 11 9 11 29/30 Read block 382 len 4096 30/30 Read block 159 len 36864 ---------------------------------------------------------------- Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Write 2 Done Done Done false 1002 Acked Write 4 Sent Done Done false 1003 Acked Flush 0 Sent Done Done false 1004 Acked Write 10 Sent Done Done false 1005 Acked Read 6 Sent Done Done false 1006 Acked Read 10 Sent Done Done false 1007 Acked Read 10 Sent Done Done false 1008 Acked Read 8 Sent Done Done false 1009 Acked Write 8 Sent Done Done false 1010 Acked Flush 0 Sent Done Done false 1011 Acked Write 6 Sent Done Done false 1012 Acked Write 10 Sent Done Done false 1013 Acked Write 6 Sent Done Done false 1014 Acked Write 2 Sent Done Done false 1015 Acked Read 8 Sent Done Done false 1016 Acked Write 3 Sent Done Done false 1017 Acked Read 5 Sent Done Done false 1018 Acked Read 7 Sent Done Done false 1019 Acked Read 5 Sent Done Done false 1020 Acked Read 7 Sent Done Done false 1021 Acked Read 3 Sent Done Done false 1022 Acked Flush 0 Sent Done Done false 1023 Acked Write 1 Sent Done Done false 1024 Acked Write 2 Sent Done Done false 1025 Acked Write 6 Sent Done Done false 1026 Acked Write 5 Sent Done Done false 1027 Acked Write 9 Sent Done Done false 1028 Acked Read 1 Sent Done Done false 1029 Acked Read 9 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 0 0 28 Done 2 30 30 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:48:12.758928231Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1829} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:12.759112846Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1829,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 172 172 286 286 286 F F F 1 020-039 170 170 170 279 279 279 F F F 2 040-059 172 172 172 286 286 286 F F F 3 060-079 172 172 172 286 286 286 F F F 4 080-099 172 172 172 286 286 286 F T T <--- 5 100-119 172 172 172 286 286 286 F F F 6 120-139 167 174 174 275 291 291 F F F <--- 7 140-159 169 174 174 278 289 289 F T T <--- 8 160-179 171 171 171 283 283 283 F F F 9 180-199 168 168 168 277 277 277 F F F 10 200-219 164 164 164 272 272 272 F F F 11 220-239 171 174 174 283 290 290 F T T <--- 12 240-259 173 174 174 288 290 290 F F F <--- 13 260-279 163 174 174 270 291 291 F F F <--- 14 280-299 170 174 174 281 291 291 F F F <--- 15 300-319 173 173 173 288 288 288 F F F 16 320-339 173 174 174 288 291 291 F F F <--- 17 340-359 171 171 171 283 283 283 F F F 18 360-379 172 172 172 286 286 286 F F F 19 380-399 173 174 174 288 289 289 T F F <--- 20 400-419 172 172 172 285 285 285 F F F 21 420-439 172 172 172 286 286 286 F F F 22 440-459 166 166 166 274 274 274 F F F 23 460-479 166 166 166 274 274 274 F F F 24 480-499 166 166 166 274 274 274 F F F 25 500-519 173 174 174 288 291 291 F T T <--- 26 520-539 173 173 173 288 288 288 F F F 27 540-559 165 174 174 273 289 289 T F F <--- 28 560-579 172 172 172 286 286 286 F F F 29 580-599 164 174 174 272 291 291 F F F <--- Max gen: 174, Max flush: 291 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 87 Verifying data now No change after verify Loop: 87 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 172 172 286 286 286 F F F 1 020-039 170 170 170 279 279 279 F F F 2 040-059 172 172 172 286 286 286 F F F 3 060-079 172 172 172 286 286 286 F F F 4 080-099 175 175 175 292 292 292 F F F 5 100-119 172 172 172 286 286 286 F F F 6 120-139 174 174 174 291 291 291 F F F 7 140-159 175 175 175 292 292 292 F F F 8 160-179 171 171 171 283 283 283 F F F 9 180-199 168 168 168 277 277 277 F F F 10 200-219 164 164 164 272 272 272 F F F 11 220-239 175 175 175 292 292 292 F F F 12 240-259 174 174 174 290 290 290 F F F 13 260-279 174 174 174 291 291 291 F F F 14 280-299 174 174 174 291 291 291 F F F 15 300-319 173 173 173 288 288 288 F F F 16 320-339 174 174 174 291 291 291 F F F 17 340-359 171 171 171 283 283 283 F F F 18 360-379 172 172 172 286 286 286 F F F 19 380-399 174 174 174 289 289 289 F F F 20 400-419 172 172 172 285 285 285 F F F 21 420-439 172 172 172 286 286 286 F F F 22 440-459 166 166 166 274 274 274 F F F 23 460-479 166 166 166 274 274 274 F F F 24 480-499 166 166 166 274 274 274 F F F 25 500-519 175 175 175 292 292 292 F F F 26 520-539 173 173 173 288 288 288 F F F 27 540-559 174 174 174 289 289 289 F F F 28 560-579 172 172 172 286 286 286 F F F 29 580-599 174 174 174 291 291 291 F F F Max gen: 175, Max flush: 292 Begin loop 88 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:17.01175553Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1837,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:17.019843326Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1837,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:17.020122368Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1837} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:17.020151897Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1837} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:17.020162627Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1837} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:17.020173977Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1837} {"msg":"Crucible 46c3aaba-fd8b-4dd8-b625-144f33178d6e has session id: 3f98a8b9-ce6d-4d46-b682-84205c4927d9","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:17.020492018Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1837,"session_id":"3f98a8b9-ce6d-4d46-b682-84205c4927d9"} {"msg":"Upstairs opts: Upstairs UUID: 46c3aaba-fd8b-4dd8-b625-144f33178d6e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:17.020507857Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1837,"session_id":"3f98a8b9-ce6d-4d46-b682-84205c4927d9"} {"msg":"Crucible stats registered with UUID: 46c3aaba-fd8b-4dd8-b625-144f33178d6e","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:17.020517627Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1837,"session_id":"3f98a8b9-ce6d-4d46-b682-84205c4927d9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:48:17.020534236Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1837,"session_id":"3f98a8b9-ce6d-4d46-b682-84205c4927d9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:17.020683642Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1837,"":"io task","client":"1","":"downstairs","session_id":"3f98a8b9-ce6d-4d46-b682-84205c4927d9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:17.020714791Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1837,"":"io task","client":"0","":"downstairs","session_id":"3f98a8b9-ce6d-4d46-b682-84205c4927d9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:17.020725721Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1837,"":"io task","client":"2","":"downstairs","session_id":"3f98a8b9-ce6d-4d46-b682-84205c4927d9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:17.020834608Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1837,"task":"crutest"} {"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:17.020849007Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1837} {"msg":"46c3aaba-fd8b-4dd8-b625-144f33178d6e active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:17.020857847Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1837,"session_id":"3f98a8b9-ce6d-4d46-b682-84205c4927d9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:17.020866827Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1837,"client":"0","":"downstairs","session_id":"3f98a8b9-ce6d-4d46-b682-84205c4927d9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:17.020876587Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1837,"client":"1","":"downstairs","session_id":"3f98a8b9-ce6d-4d46-b682-84205c4927d9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:17.020888996Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1837,"client":"2","":"downstairs","session_id":"3f98a8b9-ce6d-4d46-b682-84205c4927d9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:17.020898416Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1837,"":"io task","client":"2","":"downstairs","session_id":"3f98a8b9-ce6d-4d46-b682-84205c4927d9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:17.020908376Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1837,"":"io task","client":"0","":"downstairs","session_id":"3f98a8b9-ce6d-4d46-b682-84205c4927d9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:17.020918545Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1837,"":"io task","client":"1","":"downstairs","session_id":"3f98a8b9-ce6d-4d46-b682-84205c4927d9"} {"msg":"ds_connection connected from Ok(127.0.0.1:50670)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:17.021006693Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1837,"":"io task","client":"2","":"downstairs","session_id":"3f98a8b9-ce6d-4d46-b682-84205c4927d9"} {"msg":"ds_connection connected from Ok(127.0.0.1:55179)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:17.021032122Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1837,"":"io task","client":"0","":"downstairs","session_id":"3f98a8b9-ce6d-4d46-b682-84205c4927d9"} {"msg":"ds_connection connected from Ok(127.0.0.1:34868)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:17.02111922Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1837,"":"io task","client":"1","":"downstairs","session_id":"3f98a8b9-ce6d-4d46-b682-84205c4927d9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:17.021749511Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1837,"client":"0","":"downstairs","session_id":"3f98a8b9-ce6d-4d46-b682-84205c4927d9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:17.021835849Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1837,"client":"2","":"downstairs","session_id":"3f98a8b9-ce6d-4d46-b682-84205c4927d9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:17.021880348Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1837,"client":"1","":"downstairs","session_id":"3f98a8b9-ce6d-4d46-b682-84205c4927d9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:17.022043863Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1837,"session_id":"3f98a8b9-ce6d-4d46-b682-84205c4927d9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:17.02214108Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1837,"session_id":"3f98a8b9-ce6d-4d46-b682-84205c4927d9"} {"msg":"[0]R flush_numbers[0..12]: [286, 279, 286, 286, 292, 286, 291, 292, 283, 277, 272, 292]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:17.022196048Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1837,"":"downstairs","":"downstairs","session_id":"3f98a8b9-ce6d-4d46-b682-84205c4927d9"} {"msg":"[0]R generation[0..12]: [172, 170, 172, 172, 175, 172, 174, 175, 171, 168, 164, 175]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:17.022208108Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1837,"":"downstairs","":"downstairs","session_id":"3f98a8b9-ce6d-4d46-b682-84205c4927d9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:17.022219278Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1837,"":"downstairs","":"downstairs","session_id":"3f98a8b9-ce6d-4d46-b682-84205c4927d9"} {"msg":"[1]R flush_numbers[0..12]: [286, 279, 286, 286, 292, 286, 291, 292, 283, 277, 272, 292]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:17.022233137Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1837,"":"downstairs","":"downstairs","session_id":"3f98a8b9-ce6d-4d46-b682-84205c4927d9"} {"msg":"[1]R generation[0..12]: [172, 170, 172, 172, 175, 172, 174, 175, 171, 168, 164, 175]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:17.022243747Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1837,"":"downstairs","":"downstairs","session_id":"3f98a8b9-ce6d-4d46-b682-84205c4927d9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:17.022254647Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1837,"":"downstairs","":"downstairs","session_id":"3f98a8b9-ce6d-4d46-b682-84205c4927d9"} {"msg":"[2]R flush_numbers[0..12]: [286, 279, 286, 286, 292, 286, 291, 292, 283, 277, 272, 292]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:17.022265266Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1837,"":"downstairs","":"downstairs","session_id":"3f98a8b9-ce6d-4d46-b682-84205c4927d9"} {"msg":"[2]R generation[0..12]: [172, 170, 172, 172, 175, 172, 174, 175, 171, 168, 164, 175]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:17.022275936Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1837,"":"downstairs","":"downstairs","session_id":"3f98a8b9-ce6d-4d46-b682-84205c4927d9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:17.022286856Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1837,"":"downstairs","":"downstairs","session_id":"3f98a8b9-ce6d-4d46-b682-84205c4927d9"} {"msg":"Max found gen is 176","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:17.022296336Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1837,"":"downstairs","":"downstairs","session_id":"3f98a8b9-ce6d-4d46-b682-84205c4927d9"} {"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:17.022306115Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1837,"":"downstairs","":"downstairs","session_id":"3f98a8b9-ce6d-4d46-b682-84205c4927d9"} {"msg":"Next flush: 293","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:17.022315325Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1837,"":"downstairs","":"downstairs","session_id":"3f98a8b9-ce6d-4d46-b682-84205c4927d9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:17.022324715Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1837,"":"downstairs","":"downstairs","session_id":"3f98a8b9-ce6d-4d46-b682-84205c4927d9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:17.022334274Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1837,"session_id":"3f98a8b9-ce6d-4d46-b682-84205c4927d9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:17.022343024Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1837,"client":"0","":"downstairs","session_id":"3f98a8b9-ce6d-4d46-b682-84205c4927d9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:17.022352624Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1837,"client":"1","":"downstairs","session_id":"3f98a8b9-ce6d-4d46-b682-84205c4927d9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:17.022362344Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1837,"client":"2","":"downstairs","session_id":"3f98a8b9-ce6d-4d46-b682-84205c4927d9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:17.022372113Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1837,"session_id":"3f98a8b9-ce6d-4d46-b682-84205c4927d9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:17.022382613Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1837,"session_id":"3f98a8b9-ce6d-4d46-b682-84205c4927d9"} {"msg":"46c3aaba-fd8b-4dd8-b625-144f33178d6e is now active with session: 3f98a8b9-ce6d-4d46-b682-84205c4927d9","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:17.022392443Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1837,"session_id":"3f98a8b9-ce6d-4d46-b682-84205c4927d9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:17.022401013Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1837,"session_id":"3f98a8b9-ce6d-4d46-b682-84205c4927d9"} {"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:17.022409992Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1837} {"msg":"Activated sub_volume 46c3aaba-fd8b-4dd8-b625-144f33178d6e","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:17.022417842Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1837} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:17.022425792Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1837,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:17.022773712Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1837,"task":"crutest"} 01/30 Write block 13 len 12288 data: 11 11 11 02/30 Read block 455 len 20480 03/30 Write block 501 len 36864 data: 19 19 21 22 20 17 20 14 13 04/30 Read block 516 len 24576 05/30 Read block 510 len 8192 06/30 Write block 325 len 12288 data: 22 22 23 07/30 Read block 370 len 12288 08/30 Write block 24 len 16384 data: 10 10 7 8 09/30 Read block 83 len 24576 10/30 Read block 353 len 24576 11/30 Read block 228 len 20480 12/30 Write block 506 len 40960 data: 18 21 15 14 14 20 16 14 14 12 13/30 Flush 14/30 Flush 15/30 Write block 101 len 20480 data: 7 8 8 7 6 16/30 Write block 517 len 8192 data: 12 14 17/30 Write block 584 len 16384 data: 15 15 16 20 18/30 Read block 129 len 36864 19/30 Read block 390 len 12288 20/30 Read block 5 len 28672 21/30 Read block 506 len 40960 22/30 Read block 327 len 4096 23/30 Write block 41 len 28672 data: 10 7 6 6 9 9 8 24/30 Flush 25/30 Write block 188 len 40960 data: 12 13 13 12 12 12 14 12 13 13 26/30 Write block 176 len 24576 data: 13 13 13 15 18 15 27/30 Read block 277 len 4096 28/30 Read block 435 len 40960 29/30 Read block 322 len 36864 30/30 Read block 486 len 12288 ---------------------------------------------------------------- Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Read 5 Done Done Done false 1002 Acked Write 9 Done Done Done false 1003 Acked Read 6 Done Done Done false 1004 Acked Read 2 Done Done Done false 1005 Acked Write 3 Done Done Done false 1006 Acked Read 3 Done Sent Done false 1007 Acked Write 4 Done Sent Done false 1008 Acked Read 6 Done Sent Done false 1009 Acked Read 6 Done Sent Done false 1010 Acked Read 5 Done Sent Done false 1011 Acked Write 10 Done Sent Done false 1012 Acked Flush 0 Done Sent Done false 1013 Acked Flush 0 Done Sent Done false 1014 Acked Write 5 Done Sent Done false 1015 Acked Write 2 Done Sent Done false 1016 Acked Write 4 Done Sent Done false 1017 Acked Read 9 Done Sent Done false 1018 Acked Read 3 Done Sent Done false 1019 Acked Read 7 Done Sent Done false 1020 Acked Read 10 Done Sent Done false 1021 Acked Read 1 Done Sent Done false 1022 Acked Write 7 Done Sent Done false 1023 Acked Flush 0 Done Sent Done false 1024 Acked Write 10 Done Sent Done false 1025 Acked Write 6 Done Sent Done false 1026 Acked Read 1 Done Sent Done false 1027 Acked Read 10 Done Sent Done false 1028 Acked Read 9 Done Sent Done false 1029 Acked Read 3 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 24 0 24 Done 30 6 30 66 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) None Some(JobId(1023)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:48:17.031968047Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1837} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:17.032152791Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1837,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 176 172 176 293 286 293 F T F <--- 1 020-039 176 170 176 293 279 293 F F F <--- 2 040-059 176 172 176 295 286 295 F F F <--- 3 060-079 172 172 172 286 286 286 F F F 4 080-099 175 175 175 292 292 292 F F F 5 100-119 176 172 176 295 286 295 F F F <--- 6 120-139 174 174 174 291 291 291 F F F 7 140-159 175 175 175 292 292 292 F F F 8 160-179 171 171 171 283 283 283 T F T <--- 9 180-199 168 168 168 277 277 277 T F T <--- 10 200-219 164 164 164 272 272 272 F F F 11 220-239 175 175 175 292 292 292 F F F 12 240-259 174 174 174 290 290 290 F F F 13 260-279 174 174 174 291 291 291 F F F 14 280-299 174 174 174 291 291 291 F F F 15 300-319 173 173 173 288 288 288 F F F 16 320-339 176 174 176 293 291 293 F T F <--- 17 340-359 171 171 171 283 283 283 F F F 18 360-379 172 172 172 286 286 286 F F F 19 380-399 174 174 174 289 289 289 F F F 20 400-419 172 172 172 285 285 285 F F F 21 420-439 172 172 172 286 286 286 F F F 22 440-459 166 166 166 274 274 274 F F F 23 460-479 166 166 166 274 274 274 F F F 24 480-499 166 166 166 274 274 274 F F F 25 500-519 176 175 176 295 292 295 F T F <--- 26 520-539 173 173 173 288 288 288 F F F 27 540-559 174 174 174 289 289 289 F F F 28 560-579 172 172 172 286 286 286 F F F 29 580-599 176 174 176 295 291 295 F F F <--- Max gen: 176, Max flush: 295 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 88 Verifying data now No change after verify Loop: 88 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 176 176 176 293 293 293 F F F 1 020-039 176 176 176 293 293 293 F F F 2 040-059 176 176 176 295 295 295 F F F 3 060-079 172 172 172 286 286 286 F F F 4 080-099 175 175 175 292 292 292 F F F 5 100-119 176 176 176 295 295 295 F F F 6 120-139 174 174 174 291 291 291 F F F 7 140-159 175 175 175 292 292 292 F F F 8 160-179 177 177 177 296 296 296 F F F 9 180-199 177 177 177 296 296 296 F F F 10 200-219 164 164 164 272 272 272 F F F 11 220-239 175 175 175 292 292 292 F F F 12 240-259 174 174 174 290 290 290 F F F 13 260-279 174 174 174 291 291 291 F F F 14 280-299 174 174 174 291 291 291 F F F 15 300-319 173 173 173 288 288 288 F F F 16 320-339 176 176 176 293 293 293 F F F 17 340-359 171 171 171 283 283 283 F F F 18 360-379 172 172 172 286 286 286 F F F 19 380-399 174 174 174 289 289 289 F F F 20 400-419 172 172 172 285 285 285 F F F 21 420-439 172 172 172 286 286 286 F F F 22 440-459 166 166 166 274 274 274 F F F 23 460-479 166 166 166 274 274 274 F F F 24 480-499 166 166 166 274 274 274 F F F 25 500-519 176 176 176 295 295 295 F F F 26 520-539 173 173 173 288 288 288 F F F 27 540-559 174 174 174 289 289 289 F F F 28 560-579 172 172 172 286 286 286 F F F 29 580-599 176 176 176 295 295 295 F F F Max gen: 177, Max flush: 296 Begin loop 89 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:21.270580595Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1845,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:21.278498087Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1845,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:21.278813998Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1845} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:21.278839877Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1845} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:21.278848387Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1845} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:21.278859837Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1845} {"msg":"Crucible 9d2ecdb0-4c19-46ce-a02c-e487a136fa3e has session id: dfdf0a8e-892c-4a12-b1da-dea418b2d88b","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:21.279204037Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1845,"session_id":"dfdf0a8e-892c-4a12-b1da-dea418b2d88b"} {"msg":"Upstairs opts: Upstairs UUID: 9d2ecdb0-4c19-46ce-a02c-e487a136fa3e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:21.279232626Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1845,"session_id":"dfdf0a8e-892c-4a12-b1da-dea418b2d88b"} {"msg":"Crucible stats registered with UUID: 9d2ecdb0-4c19-46ce-a02c-e487a136fa3e","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:21.279245256Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1845,"session_id":"dfdf0a8e-892c-4a12-b1da-dea418b2d88b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:48:21.279254305Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1845,"session_id":"dfdf0a8e-892c-4a12-b1da-dea418b2d88b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:21.279404691Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1845,"":"io task","client":"2","":"downstairs","session_id":"dfdf0a8e-892c-4a12-b1da-dea418b2d88b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:21.27943567Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1845,"":"io task","client":"0","":"downstairs","session_id":"dfdf0a8e-892c-4a12-b1da-dea418b2d88b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:21.27944663Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1845,"":"io task","client":"1","":"downstairs","session_id":"dfdf0a8e-892c-4a12-b1da-dea418b2d88b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:21.279569096Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1845,"task":"crutest"} {"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:21.279588126Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1845} {"msg":"9d2ecdb0-4c19-46ce-a02c-e487a136fa3e active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:21.279619135Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1845,"session_id":"dfdf0a8e-892c-4a12-b1da-dea418b2d88b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:21.279628384Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1845,"client":"0","":"downstairs","session_id":"dfdf0a8e-892c-4a12-b1da-dea418b2d88b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:21.279637934Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1845,"client":"1","":"downstairs","session_id":"dfdf0a8e-892c-4a12-b1da-dea418b2d88b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:21.279647584Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1845,"client":"2","":"downstairs","session_id":"dfdf0a8e-892c-4a12-b1da-dea418b2d88b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:21.279657034Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1845,"":"io task","client":"2","":"downstairs","session_id":"dfdf0a8e-892c-4a12-b1da-dea418b2d88b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:21.279666983Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1845,"":"io task","client":"0","":"downstairs","session_id":"dfdf0a8e-892c-4a12-b1da-dea418b2d88b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:21.279676803Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1845,"":"io task","client":"1","":"downstairs","session_id":"dfdf0a8e-892c-4a12-b1da-dea418b2d88b"} {"msg":"ds_connection connected from Ok(127.0.0.1:36025)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:21.27977323Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1845,"":"io task","client":"1","":"downstairs","session_id":"dfdf0a8e-892c-4a12-b1da-dea418b2d88b"} {"msg":"ds_connection connected from Ok(127.0.0.1:53753)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:21.279802969Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1845,"":"io task","client":"0","":"downstairs","session_id":"dfdf0a8e-892c-4a12-b1da-dea418b2d88b"} {"msg":"ds_connection connected from Ok(127.0.0.1:62952)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:21.279840968Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1845,"":"io task","client":"2","":"downstairs","session_id":"dfdf0a8e-892c-4a12-b1da-dea418b2d88b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:21.280494449Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1845,"client":"2","":"downstairs","session_id":"dfdf0a8e-892c-4a12-b1da-dea418b2d88b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:21.280515219Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1845,"client":"1","":"downstairs","session_id":"dfdf0a8e-892c-4a12-b1da-dea418b2d88b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:21.280535598Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1845,"client":"0","":"downstairs","session_id":"dfdf0a8e-892c-4a12-b1da-dea418b2d88b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:21.2808291Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1845,"session_id":"dfdf0a8e-892c-4a12-b1da-dea418b2d88b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:21.28083908Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1845,"session_id":"dfdf0a8e-892c-4a12-b1da-dea418b2d88b"} {"msg":"[0]R flush_numbers[0..12]: [293, 293, 295, 286, 292, 295, 291, 292, 296, 296, 272, 292]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:21.280891308Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1845,"":"downstairs","":"downstairs","session_id":"dfdf0a8e-892c-4a12-b1da-dea418b2d88b"} {"msg":"[0]R generation[0..12]: [176, 176, 176, 172, 175, 176, 174, 175, 177, 177, 164, 175]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:21.280902608Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1845,"":"downstairs","":"downstairs","session_id":"dfdf0a8e-892c-4a12-b1da-dea418b2d88b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:21.280913517Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1845,"":"downstairs","":"downstairs","session_id":"dfdf0a8e-892c-4a12-b1da-dea418b2d88b"} {"msg":"[1]R flush_numbers[0..12]: [293, 293, 295, 286, 292, 295, 291, 292, 296, 296, 272, 292]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:21.280923947Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1845,"":"downstairs","":"downstairs","session_id":"dfdf0a8e-892c-4a12-b1da-dea418b2d88b"} {"msg":"[1]R generation[0..12]: [176, 176, 176, 172, 175, 176, 174, 175, 177, 177, 164, 175]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:21.280934337Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1845,"":"downstairs","":"downstairs","session_id":"dfdf0a8e-892c-4a12-b1da-dea418b2d88b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:21.280945176Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1845,"":"downstairs","":"downstairs","session_id":"dfdf0a8e-892c-4a12-b1da-dea418b2d88b"} {"msg":"[2]R flush_numbers[0..12]: [293, 293, 295, 286, 292, 295, 291, 292, 296, 296, 272, 292]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:21.280955696Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1845,"":"downstairs","":"downstairs","session_id":"dfdf0a8e-892c-4a12-b1da-dea418b2d88b"} {"msg":"[2]R generation[0..12]: [176, 176, 176, 172, 175, 176, 174, 175, 177, 177, 164, 175]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:21.280966056Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1845,"":"downstairs","":"downstairs","session_id":"dfdf0a8e-892c-4a12-b1da-dea418b2d88b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:21.280979215Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1845,"":"downstairs","":"downstairs","session_id":"dfdf0a8e-892c-4a12-b1da-dea418b2d88b"} {"msg":"Max found gen is 178","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:21.280988555Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1845,"":"downstairs","":"downstairs","session_id":"dfdf0a8e-892c-4a12-b1da-dea418b2d88b"} {"msg":"Generation requested: 178 >= found:178","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:21.280998055Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1845,"":"downstairs","":"downstairs","session_id":"dfdf0a8e-892c-4a12-b1da-dea418b2d88b"} {"msg":"Next flush: 297","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:21.281007415Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1845,"":"downstairs","":"downstairs","session_id":"dfdf0a8e-892c-4a12-b1da-dea418b2d88b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:21.281016884Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1845,"":"downstairs","":"downstairs","session_id":"dfdf0a8e-892c-4a12-b1da-dea418b2d88b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:21.281026514Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1845,"session_id":"dfdf0a8e-892c-4a12-b1da-dea418b2d88b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:21.281035234Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1845,"client":"0","":"downstairs","session_id":"dfdf0a8e-892c-4a12-b1da-dea418b2d88b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:21.281044824Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1845,"client":"1","":"downstairs","session_id":"dfdf0a8e-892c-4a12-b1da-dea418b2d88b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:21.281054293Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1845,"client":"2","":"downstairs","session_id":"dfdf0a8e-892c-4a12-b1da-dea418b2d88b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:21.281064213Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1845,"session_id":"dfdf0a8e-892c-4a12-b1da-dea418b2d88b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:21.281073203Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1845,"session_id":"dfdf0a8e-892c-4a12-b1da-dea418b2d88b"} {"msg":"9d2ecdb0-4c19-46ce-a02c-e487a136fa3e is now active with session: dfdf0a8e-892c-4a12-b1da-dea418b2d88b","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:21.281083052Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1845,"session_id":"dfdf0a8e-892c-4a12-b1da-dea418b2d88b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:21.281091602Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1845,"session_id":"dfdf0a8e-892c-4a12-b1da-dea418b2d88b"} {"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:21.281100442Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1845} {"msg":"Activated sub_volume 9d2ecdb0-4c19-46ce-a02c-e487a136fa3e","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:21.281108402Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1845} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:21.281116302Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1845,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:21.281449712Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1845,"task":"crutest"} 01/30 Flush 02/30 Write block 142 len 4096 data: 19 03/30 Write block 593 len 24576 data: 9 8 8 7 6 5 04/30 Read block 223 len 20480 05/30 Write block 314 len 16384 data: 16 15 12 12 06/30 Read block 280 len 40960 07/30 Flush 08/30 Write block 196 len 40960 data: 14 14 14 12 9 16 19 18 19 16 09/30 Read block 493 len 4096 10/30 Write block 40 len 36864 data: 9 11 8 7 7 10 10 9 10 11/30 Write block 363 len 8192 data: 19 19 12/30 Read block 186 len 36864 13/30 Write block 446 len 40960 data: 9 7 8 10 11 10 9 10 10 10 14/30 Read block 292 len 28672 15/30 Flush 16/30 Write block 461 len 32768 data: 9 9 9 8 6 6 6 9 17/30 Read block 309 len 12288 18/30 Write block 213 len 28672 data: 18 16 13 15 14 13 12 19/30 Write block 379 len 40960 data: 20 20 20 23 17 15 14 12 13 14 20/30 Flush 21/30 Write block 585 len 16384 data: 16 17 21 17 22/30 Flush 23/30 Write block 284 len 12288 data: 11 14 13 24/30 Flush 25/30 Write block 54 len 12288 data: 11 14 15 26/30 Write block 435 len 28672 data: 9 10 7 8 8 10 9 27/30 Read block 268 len 28672 28/30 Write block 588 len 24576 data: 18 11 11 10 9 10 29/30 Read block 429 len 32768 30/30 Read block 475 len 24576 ---------------------------------------------------------------- Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Flush 0 Done Sent Done false 1001 Acked Write 1 Done Sent Done false 1002 Acked Write 6 Done Sent Done false 1003 Acked Read 5 Done Sent Done false 1004 Acked Write 4 Done Sent Done false 1005 Acked Read 10 Done Sent Done false 1006 Acked Flush 0 Done Sent Done false 1007 Acked Write 10 Done Sent Done false 1008 Acked Read 1 Done Sent Done false 1009 Acked Write 9 Done Sent Done false 1010 Acked Write 2 Done Sent Done false 1011 Acked Read 9 Done Sent Done false 1012 Acked Write 10 Done Sent Done false 1013 Acked Read 7 Done Sent Done false 1014 Acked Flush 0 Done Sent Done false 1015 Acked Write 8 Done Sent Done false 1016 Acked Read 3 Done Sent Done false 1017 Acked Write 7 Done Sent Done false 1018 Acked Write 10 Done Sent Done false 1019 Acked Flush 0 Done Sent Done false 1020 Acked Write 4 Done Sent Done false 1021 Acked Flush 0 Done Sent Done false 1022 Acked Write 3 Done Sent Done false 1023 Acked Flush 0 Done Sent Done false 1024 Acked Write 3 Done Sent Done false 1025 Acked Write 7 Done Sent Done false 1026 Acked Read 7 Done Sent Done false 1027 Acked Write 6 Done Sent Done false 1028 Acked Read 8 Done Sent Done false 1029 Acked Read 6 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 30 0 30 Done 30 0 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) None Some(JobId(1023)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:48:21.290661586Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1845} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:21.290848251Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1845,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 176 176 176 293 293 293 F F F 1 020-039 176 176 176 293 293 293 F F F 2 040-059 178 176 178 299 295 299 T F T <--- 3 060-079 172 172 172 286 286 286 F F F 4 080-099 175 175 175 292 292 292 F F F 5 100-119 176 176 176 295 295 295 F F F 6 120-139 174 174 174 291 291 291 F F F 7 140-159 178 175 178 298 292 298 F F F <--- 8 160-179 177 177 177 296 296 296 F F F 9 180-199 178 177 178 299 296 299 F F F <--- 10 200-219 178 164 178 300 272 300 F F F <--- 11 220-239 175 175 175 292 292 292 F F F 12 240-259 174 174 174 290 290 290 F F F 13 260-279 174 174 174 291 291 291 F F F 14 280-299 178 174 178 302 291 302 F F F <--- 15 300-319 178 173 178 298 288 298 F F F <--- 16 320-339 176 176 176 293 293 293 F F F 17 340-359 171 171 171 283 283 283 F F F 18 360-379 178 172 178 300 286 300 F F F <--- 19 380-399 178 174 178 300 289 300 F F F <--- 20 400-419 172 172 172 285 285 285 F F F 21 420-439 172 172 172 286 286 286 T F T <--- 22 440-459 178 166 178 299 274 299 T F T <--- 23 460-479 178 166 178 300 274 300 F F F <--- 24 480-499 166 166 166 274 274 274 F F F 25 500-519 176 176 176 295 295 295 F F F 26 520-539 173 173 173 288 288 288 F F F 27 540-559 174 174 174 289 289 289 F F F 28 560-579 172 172 172 286 286 286 F F F 29 580-599 178 176 178 301 295 301 T F T <--- Max gen: 178, Max flush: 302 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 89 Verifying data now No change after verify Loop: 89 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 176 176 176 293 293 293 F F F 1 020-039 176 176 176 293 293 293 F F F 2 040-059 179 179 179 303 303 303 F F F 3 060-079 172 172 172 286 286 286 F F F 4 080-099 175 175 175 292 292 292 F F F 5 100-119 176 176 176 295 295 295 F F F 6 120-139 174 174 174 291 291 291 F F F 7 140-159 178 178 178 298 298 298 F F F 8 160-179 177 177 177 296 296 296 F F F 9 180-199 178 178 178 299 299 299 F F F 10 200-219 178 178 178 300 300 300 F F F 11 220-239 175 175 175 292 292 292 F F F 12 240-259 174 174 174 290 290 290 F F F 13 260-279 174 174 174 291 291 291 F F F 14 280-299 178 178 178 302 302 302 F F F 15 300-319 178 178 178 298 298 298 F F F 16 320-339 176 176 176 293 293 293 F F F 17 340-359 171 171 171 283 283 283 F F F 18 360-379 178 178 178 300 300 300 F F F 19 380-399 178 178 178 300 300 300 F F F 20 400-419 172 172 172 285 285 285 F F F 21 420-439 179 179 179 303 303 303 F F F 22 440-459 179 179 179 303 303 303 F F F 23 460-479 178 178 178 300 300 300 F F F 24 480-499 166 166 166 274 274 274 F F F 25 500-519 176 176 176 295 295 295 F F F 26 520-539 173 173 173 288 288 288 F F F 27 540-559 174 174 174 289 289 289 F F F 28 560-579 172 172 172 286 286 286 F F F 29 580-599 179 179 179 303 303 303 F F F Max gen: 179, Max flush: 303 Begin loop 90 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:25.548426913Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1853,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:25.556452091Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1853,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:25.556748922Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1853} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:25.556772702Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1853} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:25.556781322Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1853} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:25.556792581Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1853} {"msg":"Crucible a2818729-461f-44da-be57-d024093f9de0 has session id: 829a2f01-7180-43b1-9008-3609cfc87e8b","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:25.55718822Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1853,"session_id":"829a2f01-7180-43b1-9008-3609cfc87e8b"} {"msg":"Upstairs opts: Upstairs UUID: a2818729-461f-44da-be57-d024093f9de0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:25.557209129Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1853,"session_id":"829a2f01-7180-43b1-9008-3609cfc87e8b"} {"msg":"Crucible stats registered with UUID: a2818729-461f-44da-be57-d024093f9de0","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:25.557219239Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1853,"session_id":"829a2f01-7180-43b1-9008-3609cfc87e8b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:48:25.557228359Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1853,"session_id":"829a2f01-7180-43b1-9008-3609cfc87e8b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:25.557377974Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1853,"":"io task","client":"0","":"downstairs","session_id":"829a2f01-7180-43b1-9008-3609cfc87e8b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:25.557408633Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1853,"":"io task","client":"2","":"downstairs","session_id":"829a2f01-7180-43b1-9008-3609cfc87e8b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:25.557419563Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1853,"":"io task","client":"1","":"downstairs","session_id":"829a2f01-7180-43b1-9008-3609cfc87e8b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:25.557594388Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1853,"task":"crutest"} {"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:25.557608988Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1853} {"msg":"a2818729-461f-44da-be57-d024093f9de0 active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:25.557626237Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1853,"session_id":"829a2f01-7180-43b1-9008-3609cfc87e8b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:25.557635097Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1853,"client":"0","":"downstairs","session_id":"829a2f01-7180-43b1-9008-3609cfc87e8b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:25.557644707Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1853,"client":"1","":"downstairs","session_id":"829a2f01-7180-43b1-9008-3609cfc87e8b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:25.557654306Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1853,"client":"2","":"downstairs","session_id":"829a2f01-7180-43b1-9008-3609cfc87e8b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:25.557663696Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1853,"":"io task","client":"2","":"downstairs","session_id":"829a2f01-7180-43b1-9008-3609cfc87e8b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:25.557673616Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1853,"":"io task","client":"0","":"downstairs","session_id":"829a2f01-7180-43b1-9008-3609cfc87e8b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:25.557683446Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1853,"":"io task","client":"1","":"downstairs","session_id":"829a2f01-7180-43b1-9008-3609cfc87e8b"} {"msg":"ds_connection connected from Ok(127.0.0.1:43771)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:25.557743184Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1853,"":"io task","client":"1","":"downstairs","session_id":"829a2f01-7180-43b1-9008-3609cfc87e8b"} {"msg":"ds_connection connected from Ok(127.0.0.1:45492)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:25.557777983Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1853,"":"io task","client":"0","":"downstairs","session_id":"829a2f01-7180-43b1-9008-3609cfc87e8b"} {"msg":"ds_connection connected from Ok(127.0.0.1:55801)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:25.557833951Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1853,"":"io task","client":"2","":"downstairs","session_id":"829a2f01-7180-43b1-9008-3609cfc87e8b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:25.558549661Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1853,"client":"2","":"downstairs","session_id":"829a2f01-7180-43b1-9008-3609cfc87e8b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:25.5585753Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1853,"client":"0","":"downstairs","session_id":"829a2f01-7180-43b1-9008-3609cfc87e8b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:25.558587159Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1853,"client":"1","":"downstairs","session_id":"829a2f01-7180-43b1-9008-3609cfc87e8b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:25.55891331Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1853,"session_id":"829a2f01-7180-43b1-9008-3609cfc87e8b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:25.558957589Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1853,"session_id":"829a2f01-7180-43b1-9008-3609cfc87e8b"} {"msg":"[0]R flush_numbers[0..12]: [293, 293, 303, 286, 292, 295, 291, 298, 296, 299, 300, 292]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:25.559017457Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1853,"":"downstairs","":"downstairs","session_id":"829a2f01-7180-43b1-9008-3609cfc87e8b"} {"msg":"[0]R generation[0..12]: [176, 176, 179, 172, 175, 176, 174, 178, 177, 178, 178, 175]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:25.559028537Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1853,"":"downstairs","":"downstairs","session_id":"829a2f01-7180-43b1-9008-3609cfc87e8b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:25.559039526Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1853,"":"downstairs","":"downstairs","session_id":"829a2f01-7180-43b1-9008-3609cfc87e8b"} {"msg":"[1]R flush_numbers[0..12]: [293, 293, 303, 286, 292, 295, 291, 298, 296, 299, 300, 292]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:25.559050116Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1853,"":"downstairs","":"downstairs","session_id":"829a2f01-7180-43b1-9008-3609cfc87e8b"} {"msg":"[1]R generation[0..12]: [176, 176, 179, 172, 175, 176, 174, 178, 177, 178, 178, 175]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:25.559060676Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1853,"":"downstairs","":"downstairs","session_id":"829a2f01-7180-43b1-9008-3609cfc87e8b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:25.559071485Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1853,"":"downstairs","":"downstairs","session_id":"829a2f01-7180-43b1-9008-3609cfc87e8b"} {"msg":"[2]R flush_numbers[0..12]: [293, 293, 303, 286, 292, 295, 291, 298, 296, 299, 300, 292]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:25.559081935Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1853,"":"downstairs","":"downstairs","session_id":"829a2f01-7180-43b1-9008-3609cfc87e8b"} {"msg":"[2]R generation[0..12]: [176, 176, 179, 172, 175, 176, 174, 178, 177, 178, 178, 175]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:25.559092305Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1853,"":"downstairs","":"downstairs","session_id":"829a2f01-7180-43b1-9008-3609cfc87e8b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:25.559102945Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1853,"":"downstairs","":"downstairs","session_id":"829a2f01-7180-43b1-9008-3609cfc87e8b"} {"msg":"Max found gen is 180","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:25.559112434Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1853,"":"downstairs","":"downstairs","session_id":"829a2f01-7180-43b1-9008-3609cfc87e8b"} {"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:25.559121954Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1853,"":"downstairs","":"downstairs","session_id":"829a2f01-7180-43b1-9008-3609cfc87e8b"} {"msg":"Next flush: 304","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:25.559131154Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1853,"":"downstairs","":"downstairs","session_id":"829a2f01-7180-43b1-9008-3609cfc87e8b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:25.559140384Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1853,"":"downstairs","":"downstairs","session_id":"829a2f01-7180-43b1-9008-3609cfc87e8b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:25.559151643Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1853,"session_id":"829a2f01-7180-43b1-9008-3609cfc87e8b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:25.559160323Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1853,"client":"0","":"downstairs","session_id":"829a2f01-7180-43b1-9008-3609cfc87e8b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:25.559169793Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1853,"client":"1","":"downstairs","session_id":"829a2f01-7180-43b1-9008-3609cfc87e8b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:25.559179322Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1853,"client":"2","":"downstairs","session_id":"829a2f01-7180-43b1-9008-3609cfc87e8b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:25.559188982Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1853,"session_id":"829a2f01-7180-43b1-9008-3609cfc87e8b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:25.559197972Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1853,"session_id":"829a2f01-7180-43b1-9008-3609cfc87e8b"} {"msg":"a2818729-461f-44da-be57-d024093f9de0 is now active with session: 829a2f01-7180-43b1-9008-3609cfc87e8b","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:25.559207762Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1853,"session_id":"829a2f01-7180-43b1-9008-3609cfc87e8b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:25.559216451Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1853,"session_id":"829a2f01-7180-43b1-9008-3609cfc87e8b"} {"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:25.559225531Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1853} {"msg":"Activated sub_volume a2818729-461f-44da-be57-d024093f9de0","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:25.559233421Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1853} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:25.559241441Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1853,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:25.559530532Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1853,"task":"crutest"} 01/30 Write block 77 len 12288 data: 17 19 17 02/30 Write block 170 len 20480 data: 14 14 17 18 15 03/30 Write block 0 len 20480 data: 5 7 10 7 12 04/30 Flush 05/30 Write block 161 len 4096 data: 11 06/30 Read block 36 len 40960 07/30 Read block 280 len 36864 08/30 Read block 170 len 24576 09/30 Read block 436 len 4096 10/30 Write block 357 len 32768 data: 14 13 13 14 16 19 20 20 11/30 Write block 108 len 32768 data: 6 6 7 10 15 15 13 14 12/30 Read block 439 len 36864 13/30 Write block 274 len 12288 data: 8 9 12 14/30 Flush 15/30 Write block 400 len 36864 data: 17 16 17 15 16 16 13 12 8 16/30 Write block 220 len 36864 data: 10 10 12 16 17 19 19 20 17 17/30 Write block 577 len 36864 data: 13 14 13 13 13 15 16 16 17 18/30 Read block 315 len 40960 19/30 Flush 20/30 Write block 60 len 28672 data: 13 15 15 11 9 8 7 21/30 Read block 324 len 32768 22/30 Read block 106 len 8192 23/30 Read block 332 len 40960 24/30 Write block 107 len 24576 data: 8 7 7 8 11 16 25/30 Read block 362 len 4096 26/30 Write block 547 len 20480 data: 12 14 17 15 10 27/30 Read block 184 len 28672 28/30 Write block 260 len 28672 data: 13 12 13 11 8 8 9 29/30 Write block 432 len 20480 data: 11 8 6 10 11 30/30 Write block 533 len 28672 data: 15 15 19 17 20 22 22 ---------------------------------------------------------------- Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Write 5 Done Done Sent false 1002 Acked Write 5 Done Done Sent false 1003 Acked Flush 0 Done Done Sent false 1004 Acked Write 1 Done Done Sent false 1005 Acked Read 10 Done Done Sent false 1006 Acked Read 9 Done Done Sent false 1007 Acked Read 6 Done Done Sent false 1008 Acked Read 1 Done Done Sent false 1009 Acked Write 8 Done Done Sent false 1010 Acked Write 8 Done Done Sent false 1011 Acked Read 9 Done Done Sent false 1012 Acked Write 3 Done Done Sent false 1013 Acked Flush 0 Done Done Sent false 1014 Acked Write 9 Done Done Sent false 1015 Acked Write 9 Done Done Sent false 1016 Acked Write 9 Done Done Sent false 1017 Acked Read 10 Done Done Sent false 1018 Acked Flush 0 Done Done Sent false 1019 Acked Write 7 Done Done Sent false 1020 Acked Read 8 Done Done Sent false 1021 Acked Read 2 Done Done Sent false 1022 Acked Read 10 Done Done Sent false 1023 Acked Write 6 Done Done Sent false 1024 Acked Read 1 Done Done Sent false 1025 Acked Write 5 Done Done Sent false 1026 Acked Read 7 Done Done Sent false 1027 Acked Write 7 Done Done Sent false 1028 Acked Write 5 Done Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 29 32 Done 29 28 1 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:48:25.56829261Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1853} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:25.568510173Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1853,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 180 180 176 304 304 293 F F F <--- 1 020-039 176 176 176 293 293 293 F F F 2 040-059 179 179 179 303 303 303 F F F 3 060-079 180 180 172 304 304 286 T T T <--- 4 080-099 175 175 175 292 292 292 F F F 5 100-119 180 180 176 305 305 295 T T F <--- 6 120-139 174 174 174 291 291 291 F F F 7 140-159 178 178 178 298 298 298 F F F 8 160-179 180 180 177 305 305 296 F F F <--- 9 180-199 178 178 178 299 299 299 F F F 10 200-219 178 178 178 300 300 300 F F F 11 220-239 180 180 175 306 306 292 F F F <--- 12 240-259 174 174 174 290 290 290 F F F 13 260-279 180 180 174 305 305 291 T T F <--- 14 280-299 178 178 178 302 302 302 F F F 15 300-319 178 178 178 298 298 298 F F F 16 320-339 176 176 176 293 293 293 F F F 17 340-359 180 180 171 305 305 283 F F F <--- 18 360-379 180 180 178 305 305 300 F F F <--- 19 380-399 178 178 178 300 300 300 F F F 20 400-419 180 180 172 306 306 285 F F F <--- 21 420-439 179 179 179 303 303 303 T T F <--- 22 440-459 179 179 179 303 303 303 F F F 23 460-479 178 178 178 300 300 300 F F F 24 480-499 166 166 166 274 274 274 F F F 25 500-519 176 176 176 295 295 295 F F F 26 520-539 173 173 173 288 288 288 T T F <--- 27 540-559 174 174 174 289 289 289 T T F <--- 28 560-579 180 180 172 306 306 286 F F F <--- 29 580-599 180 180 179 306 306 303 F F F <--- Max gen: 180, Max flush: 306 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 90 Verifying data now No change after verify Loop: 90 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 180 180 180 304 304 304 F F F 1 020-039 176 176 176 293 293 293 F F F 2 040-059 179 179 179 303 303 303 F F F 3 060-079 181 181 181 307 307 307 F F F 4 080-099 175 175 175 292 292 292 F F F 5 100-119 181 181 181 307 307 307 F F F 6 120-139 174 174 174 291 291 291 F F F 7 140-159 178 178 178 298 298 298 F F F 8 160-179 180 180 180 305 305 305 F F F 9 180-199 178 178 178 299 299 299 F F F 10 200-219 178 178 178 300 300 300 F F F 11 220-239 180 180 180 306 306 306 F F F 12 240-259 174 174 174 290 290 290 F F F 13 260-279 181 181 181 307 307 307 F F F 14 280-299 178 178 178 302 302 302 F F F 15 300-319 178 178 178 298 298 298 F F F 16 320-339 176 176 176 293 293 293 F F F 17 340-359 180 180 180 305 305 305 F F F 18 360-379 180 180 180 305 305 305 F F F 19 380-399 178 178 178 300 300 300 F F F 20 400-419 180 180 180 306 306 306 F F F 21 420-439 181 181 181 307 307 307 F F F 22 440-459 179 179 179 303 303 303 F F F 23 460-479 178 178 178 300 300 300 F F F 24 480-499 166 166 166 274 274 274 F F F 25 500-519 176 176 176 295 295 295 F F F 26 520-539 181 181 181 307 307 307 F F F 27 540-559 181 181 181 307 307 307 F F F 28 560-579 180 180 180 306 306 306 F F F 29 580-599 180 180 180 306 306 306 F F F Max gen: 181, Max flush: 307 Begin loop 91 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:29.819745647Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1861,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:29.827745137Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1861,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:29.828052788Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1861} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:29.828077927Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1861} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:29.828088877Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1861} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:29.828100206Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1861} {"msg":"Crucible 8a7036f3-bf47-4f81-bc00-8a571b533288 has session id: a39c5c30-c4b0-4ce2-ad43-547ce9013015","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:29.828487985Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1861,"session_id":"a39c5c30-c4b0-4ce2-ad43-547ce9013015"} {"msg":"Upstairs opts: Upstairs UUID: 8a7036f3-bf47-4f81-bc00-8a571b533288, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:29.828518964Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1861,"session_id":"a39c5c30-c4b0-4ce2-ad43-547ce9013015"} {"msg":"Crucible stats registered with UUID: 8a7036f3-bf47-4f81-bc00-8a571b533288","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:29.828529204Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1861,"session_id":"a39c5c30-c4b0-4ce2-ad43-547ce9013015"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:48:29.828538304Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1861,"session_id":"a39c5c30-c4b0-4ce2-ad43-547ce9013015"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:29.82867914Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1861,"":"io task","client":"1","":"downstairs","session_id":"a39c5c30-c4b0-4ce2-ad43-547ce9013015"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:29.828711139Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1861,"":"io task","client":"0","":"downstairs","session_id":"a39c5c30-c4b0-4ce2-ad43-547ce9013015"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:29.828722019Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1861,"":"io task","client":"2","":"downstairs","session_id":"a39c5c30-c4b0-4ce2-ad43-547ce9013015"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:29.828837535Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1861,"task":"crutest"} {"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:29.828856025Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1861} {"msg":"8a7036f3-bf47-4f81-bc00-8a571b533288 active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:29.828882844Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1861,"session_id":"a39c5c30-c4b0-4ce2-ad43-547ce9013015"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:29.828892244Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1861,"client":"0","":"downstairs","session_id":"a39c5c30-c4b0-4ce2-ad43-547ce9013015"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:29.828901953Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1861,"client":"1","":"downstairs","session_id":"a39c5c30-c4b0-4ce2-ad43-547ce9013015"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:29.828913363Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1861,"client":"2","":"downstairs","session_id":"a39c5c30-c4b0-4ce2-ad43-547ce9013015"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:29.828922893Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1861,"":"io task","client":"2","":"downstairs","session_id":"a39c5c30-c4b0-4ce2-ad43-547ce9013015"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:29.828951102Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1861,"":"io task","client":"0","":"downstairs","session_id":"a39c5c30-c4b0-4ce2-ad43-547ce9013015"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:29.828973701Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1861,"":"io task","client":"1","":"downstairs","session_id":"a39c5c30-c4b0-4ce2-ad43-547ce9013015"} {"msg":"ds_connection connected from Ok(127.0.0.1:58462)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:29.828984611Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1861,"":"io task","client":"2","":"downstairs","session_id":"a39c5c30-c4b0-4ce2-ad43-547ce9013015"} {"msg":"ds_connection connected from Ok(127.0.0.1:47640)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:29.82900675Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1861,"":"io task","client":"0","":"downstairs","session_id":"a39c5c30-c4b0-4ce2-ad43-547ce9013015"} {"msg":"ds_connection connected from Ok(127.0.0.1:55132)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:29.82901728Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1861,"":"io task","client":"1","":"downstairs","session_id":"a39c5c30-c4b0-4ce2-ad43-547ce9013015"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:29.829645492Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1861,"client":"1","":"downstairs","session_id":"a39c5c30-c4b0-4ce2-ad43-547ce9013015"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:29.82970643Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1861,"client":"0","":"downstairs","session_id":"a39c5c30-c4b0-4ce2-ad43-547ce9013015"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:29.82972769Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1861,"client":"2","":"downstairs","session_id":"a39c5c30-c4b0-4ce2-ad43-547ce9013015"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:29.829991692Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1861,"session_id":"a39c5c30-c4b0-4ce2-ad43-547ce9013015"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:29.830024231Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1861,"session_id":"a39c5c30-c4b0-4ce2-ad43-547ce9013015"} {"msg":"[0]R flush_numbers[0..12]: [304, 293, 303, 307, 292, 307, 291, 298, 305, 299, 300, 306]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:29.830079599Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1861,"":"downstairs","":"downstairs","session_id":"a39c5c30-c4b0-4ce2-ad43-547ce9013015"} {"msg":"[0]R generation[0..12]: [180, 176, 179, 181, 175, 181, 174, 178, 180, 178, 178, 180]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:29.830091009Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1861,"":"downstairs","":"downstairs","session_id":"a39c5c30-c4b0-4ce2-ad43-547ce9013015"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:29.830102099Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1861,"":"downstairs","":"downstairs","session_id":"a39c5c30-c4b0-4ce2-ad43-547ce9013015"} {"msg":"[1]R flush_numbers[0..12]: [304, 293, 303, 307, 292, 307, 291, 298, 305, 299, 300, 306]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:29.830115328Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1861,"":"downstairs","":"downstairs","session_id":"a39c5c30-c4b0-4ce2-ad43-547ce9013015"} {"msg":"[1]R generation[0..12]: [180, 176, 179, 181, 175, 181, 174, 178, 180, 178, 178, 180]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:29.830125958Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1861,"":"downstairs","":"downstairs","session_id":"a39c5c30-c4b0-4ce2-ad43-547ce9013015"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:29.830136708Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1861,"":"downstairs","":"downstairs","session_id":"a39c5c30-c4b0-4ce2-ad43-547ce9013015"} {"msg":"[2]R flush_numbers[0..12]: [304, 293, 303, 307, 292, 307, 291, 298, 305, 299, 300, 306]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:29.830147227Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1861,"":"downstairs","":"downstairs","session_id":"a39c5c30-c4b0-4ce2-ad43-547ce9013015"} {"msg":"[2]R generation[0..12]: [180, 176, 179, 181, 175, 181, 174, 178, 180, 178, 178, 180]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:29.830157617Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1861,"":"downstairs","":"downstairs","session_id":"a39c5c30-c4b0-4ce2-ad43-547ce9013015"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:29.830168337Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1861,"":"downstairs","":"downstairs","session_id":"a39c5c30-c4b0-4ce2-ad43-547ce9013015"} {"msg":"Max found gen is 182","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:29.830177557Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1861,"":"downstairs","":"downstairs","session_id":"a39c5c30-c4b0-4ce2-ad43-547ce9013015"} {"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:29.830187336Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1861,"":"downstairs","":"downstairs","session_id":"a39c5c30-c4b0-4ce2-ad43-547ce9013015"} {"msg":"Next flush: 308","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:29.830196796Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1861,"":"downstairs","":"downstairs","session_id":"a39c5c30-c4b0-4ce2-ad43-547ce9013015"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:29.830206076Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1861,"":"downstairs","":"downstairs","session_id":"a39c5c30-c4b0-4ce2-ad43-547ce9013015"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:29.830215645Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1861,"session_id":"a39c5c30-c4b0-4ce2-ad43-547ce9013015"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:29.830224315Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1861,"client":"0","":"downstairs","session_id":"a39c5c30-c4b0-4ce2-ad43-547ce9013015"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:29.830233855Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1861,"client":"1","":"downstairs","session_id":"a39c5c30-c4b0-4ce2-ad43-547ce9013015"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:29.830243325Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1861,"client":"2","":"downstairs","session_id":"a39c5c30-c4b0-4ce2-ad43-547ce9013015"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:29.830253174Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1861,"session_id":"a39c5c30-c4b0-4ce2-ad43-547ce9013015"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:29.830263644Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1861,"session_id":"a39c5c30-c4b0-4ce2-ad43-547ce9013015"} {"msg":"8a7036f3-bf47-4f81-bc00-8a571b533288 is now active with session: a39c5c30-c4b0-4ce2-ad43-547ce9013015","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:29.830273334Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1861,"session_id":"a39c5c30-c4b0-4ce2-ad43-547ce9013015"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:29.830281954Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1861,"session_id":"a39c5c30-c4b0-4ce2-ad43-547ce9013015"} {"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:29.830290963Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1861} {"msg":"Activated sub_volume 8a7036f3-bf47-4f81-bc00-8a571b533288","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:29.830298883Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1861} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:29.830306783Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1861,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:29.830647213Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1861,"task":"crutest"} 01/30 Write block 581 len 16384 data: 14 16 17 17 02/30 Write block 205 len 8192 data: 17 15 03/30 Read block 338 len 36864 04/30 Read block 350 len 28672 05/30 Write block 118 len 16384 data: 13 13 11 12 06/30 Flush 07/30 Write block 308 len 36864 data: 9 12 14 14 14 16 17 16 13 08/30 Write block 188 len 28672 data: 13 14 14 13 13 13 15 09/30 Read block 343 len 4096 10/30 Read block 359 len 4096 11/30 Read block 350 len 8192 12/30 Write block 315 len 16384 data: 17 14 13 8 13/30 Read block 343 len 4096 14/30 Write block 581 len 4096 data: 15 15/30 Write block 256 len 16384 data: 16 13 14 13 16/30 Read block 520 len 28672 17/30 Read block 186 len 28672 18/30 Read block 367 len 20480 19/30 Flush 20/30 Write block 397 len 24576 data: 13 11 16 18 17 18 21/30 Flush 22/30 Write block 379 len 24576 data: 21 21 21 24 18 16 23/30 Write block 488 len 8192 data: 20 20 24/30 Write block 509 len 24576 data: 15 15 21 17 15 15 25/30 Read block 377 len 20480 26/30 Read block 197 len 20480 27/30 Write block 88 len 28672 data: 10 12 12 10 10 12 14 28/30 Write block 146 len 32768 data: 18 17 15 10 10 10 9 8 29/30 Read block 393 len 32768 30/30 Read block 335 len 24576 ---------------------------------------------------------------- Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Sent false 1001 Acked Write 2 Done Done Sent false 1002 Acked Read 9 Done Done Sent false 1003 Acked Read 7 Done Done Sent false 1004 Acked Write 4 Done Done Sent false 1005 Acked Flush 0 Done Done Sent false 1006 Acked Write 9 Done Done Sent false 1007 Acked Write 7 Done Done Sent false 1008 Acked Read 1 Done Done Sent false 1009 Acked Read 1 Done Done Sent false 1010 Acked Read 2 Done Done Sent false 1011 Acked Write 4 Done Done Sent false 1012 Acked Read 1 Done Done Sent false 1013 Acked Write 1 Done Done Sent false 1014 Acked Write 4 Done Done Sent false 1015 Acked Read 7 Done Done Sent false 1016 Acked Read 7 Done Done Sent false 1017 Acked Read 5 Done Done Sent false 1018 Acked Flush 0 Done Done Sent false 1019 Acked Write 6 Done Done Sent false 1020 Acked Flush 0 Done Done Sent false 1021 Acked Write 6 Done Done Sent false 1022 Acked Write 2 Done Done Sent false 1023 Acked Write 6 Done Done Sent false 1024 Acked Read 5 Done Done Sent false 1025 Acked Read 5 Done Done Sent false 1026 Acked Write 7 Done Done Sent false 1027 Acked Write 8 Done Done Sent false 1028 Acked Read 8 Done Done Sent false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 30 31 Done 30 29 0 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:48:29.838503676Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1861} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:29.838703781Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1861,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 180 180 180 304 304 304 F F F 1 020-039 176 176 176 293 293 293 F F F 2 040-059 179 179 179 303 303 303 F F F 3 060-079 181 181 181 307 307 307 F F F 4 080-099 175 175 175 292 292 292 T T F <--- 5 100-119 182 182 181 308 308 307 F F F <--- 6 120-139 182 182 174 308 308 291 F F F <--- 7 140-159 178 178 178 298 298 298 T T F <--- 8 160-179 180 180 180 305 305 305 F F F 9 180-199 182 182 178 309 309 299 F F F <--- 10 200-219 182 182 178 308 308 300 F F F <--- 11 220-239 180 180 180 306 306 306 F F F 12 240-259 182 182 174 309 309 290 F F F <--- 13 260-279 181 181 181 307 307 307 F F F 14 280-299 178 178 178 302 302 302 F F F 15 300-319 182 182 178 309 309 298 F F F <--- 16 320-339 176 176 176 293 293 293 F F F 17 340-359 180 180 180 305 305 305 F F F 18 360-379 180 180 180 305 305 305 T T F <--- 19 380-399 182 182 178 310 310 300 T T F <--- 20 400-419 182 182 180 310 310 306 F F F <--- 21 420-439 181 181 181 307 307 307 F F F 22 440-459 179 179 179 303 303 303 F F F 23 460-479 178 178 178 300 300 300 F F F 24 480-499 166 166 166 274 274 274 T T F <--- 25 500-519 176 176 176 295 295 295 T T F <--- 26 520-539 181 181 181 307 307 307 F F F 27 540-559 181 181 181 307 307 307 F F F 28 560-579 180 180 180 306 306 306 F F F 29 580-599 182 182 180 309 309 306 F F F <--- Max gen: 182, Max flush: 310 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 91 Verifying data now No change after verify Loop: 91 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 180 180 180 304 304 304 F F F 1 020-039 176 176 176 293 293 293 F F F 2 040-059 179 179 179 303 303 303 F F F 3 060-079 181 181 181 307 307 307 F F F 4 080-099 183 183 183 311 311 311 F F F 5 100-119 182 182 182 308 308 308 F F F 6 120-139 182 182 182 308 308 308 F F F 7 140-159 183 183 183 311 311 311 F F F 8 160-179 180 180 180 305 305 305 F F F 9 180-199 182 182 182 309 309 309 F F F 10 200-219 182 182 182 308 308 308 F F F 11 220-239 180 180 180 306 306 306 F F F 12 240-259 182 182 182 309 309 309 F F F 13 260-279 181 181 181 307 307 307 F F F 14 280-299 178 178 178 302 302 302 F F F 15 300-319 182 182 182 309 309 309 F F F 16 320-339 176 176 176 293 293 293 F F F 17 340-359 180 180 180 305 305 305 F F F 18 360-379 183 183 183 311 311 311 F F F 19 380-399 183 183 183 311 311 311 F F F 20 400-419 182 182 182 310 310 310 F F F 21 420-439 181 181 181 307 307 307 F F F 22 440-459 179 179 179 303 303 303 F F F 23 460-479 178 178 178 300 300 300 F F F 24 480-499 183 183 183 311 311 311 F F F 25 500-519 183 183 183 311 311 311 F F F 26 520-539 181 181 181 307 307 307 F F F 27 540-559 181 181 181 307 307 307 F F F 28 560-579 180 180 180 306 306 306 F F F 29 580-599 182 182 182 309 309 309 F F F Max gen: 183, Max flush: 311 Begin loop 92 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:34.085480894Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1869,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:34.093438824Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1869,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:34.093723576Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1869} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:34.093747585Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1869} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:34.093756125Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1869} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:34.093767545Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1869} {"msg":"Crucible 21ed9e98-0739-4f31-b34e-8fc0f4775edc has session id: aebeab86-ba4f-4eb8-bba6-a10f5cf0cea7","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:34.094159964Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1869,"session_id":"aebeab86-ba4f-4eb8-bba6-a10f5cf0cea7"} {"msg":"Upstairs opts: Upstairs UUID: 21ed9e98-0739-4f31-b34e-8fc0f4775edc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:34.094180643Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1869,"session_id":"aebeab86-ba4f-4eb8-bba6-a10f5cf0cea7"} {"msg":"Crucible stats registered with UUID: 21ed9e98-0739-4f31-b34e-8fc0f4775edc","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:34.094192703Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1869,"session_id":"aebeab86-ba4f-4eb8-bba6-a10f5cf0cea7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:48:34.094201922Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1869,"session_id":"aebeab86-ba4f-4eb8-bba6-a10f5cf0cea7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:34.094350768Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1869,"":"io task","client":"0","":"downstairs","session_id":"aebeab86-ba4f-4eb8-bba6-a10f5cf0cea7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:34.094383777Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1869,"":"io task","client":"1","":"downstairs","session_id":"aebeab86-ba4f-4eb8-bba6-a10f5cf0cea7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:34.094394107Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1869,"":"io task","client":"2","":"downstairs","session_id":"aebeab86-ba4f-4eb8-bba6-a10f5cf0cea7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:34.094545062Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1869,"task":"crutest"} {"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:34.094556552Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1869} {"msg":"21ed9e98-0739-4f31-b34e-8fc0f4775edc active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:34.094572002Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1869,"session_id":"aebeab86-ba4f-4eb8-bba6-a10f5cf0cea7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:34.094580781Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1869,"client":"0","":"downstairs","session_id":"aebeab86-ba4f-4eb8-bba6-a10f5cf0cea7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:34.094590531Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1869,"client":"1","":"downstairs","session_id":"aebeab86-ba4f-4eb8-bba6-a10f5cf0cea7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:34.094600071Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1869,"client":"2","":"downstairs","session_id":"aebeab86-ba4f-4eb8-bba6-a10f5cf0cea7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:34.094609591Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1869,"":"io task","client":"2","":"downstairs","session_id":"aebeab86-ba4f-4eb8-bba6-a10f5cf0cea7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:34.09461938Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1869,"":"io task","client":"0","":"downstairs","session_id":"aebeab86-ba4f-4eb8-bba6-a10f5cf0cea7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:34.0946298Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1869,"":"io task","client":"1","":"downstairs","session_id":"aebeab86-ba4f-4eb8-bba6-a10f5cf0cea7"} {"msg":"ds_connection connected from Ok(127.0.0.1:64119)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:34.094655839Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1869,"":"io task","client":"2","":"downstairs","session_id":"aebeab86-ba4f-4eb8-bba6-a10f5cf0cea7"} {"msg":"ds_connection connected from Ok(127.0.0.1:65076)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:34.094700878Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1869,"":"io task","client":"1","":"downstairs","session_id":"aebeab86-ba4f-4eb8-bba6-a10f5cf0cea7"} {"msg":"ds_connection connected from Ok(127.0.0.1:58832)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:34.094728087Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1869,"":"io task","client":"0","":"downstairs","session_id":"aebeab86-ba4f-4eb8-bba6-a10f5cf0cea7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:34.095344479Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1869,"client":"1","":"downstairs","session_id":"aebeab86-ba4f-4eb8-bba6-a10f5cf0cea7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:34.095362559Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1869,"client":"0","":"downstairs","session_id":"aebeab86-ba4f-4eb8-bba6-a10f5cf0cea7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:34.095461426Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1869,"client":"2","":"downstairs","session_id":"aebeab86-ba4f-4eb8-bba6-a10f5cf0cea7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:34.09568224Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1869,"session_id":"aebeab86-ba4f-4eb8-bba6-a10f5cf0cea7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:34.095692119Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1869,"session_id":"aebeab86-ba4f-4eb8-bba6-a10f5cf0cea7"} {"msg":"[0]R flush_numbers[0..12]: [304, 293, 303, 307, 311, 308, 308, 311, 305, 309, 308, 306]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:34.095758427Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1869,"":"downstairs","":"downstairs","session_id":"aebeab86-ba4f-4eb8-bba6-a10f5cf0cea7"} {"msg":"[0]R generation[0..12]: [180, 176, 179, 181, 183, 182, 182, 183, 180, 182, 182, 180]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:34.095769707Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1869,"":"downstairs","":"downstairs","session_id":"aebeab86-ba4f-4eb8-bba6-a10f5cf0cea7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:34.095780847Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1869,"":"downstairs","":"downstairs","session_id":"aebeab86-ba4f-4eb8-bba6-a10f5cf0cea7"} {"msg":"[1]R flush_numbers[0..12]: [304, 293, 303, 307, 311, 308, 308, 311, 305, 309, 308, 306]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:34.095791407Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1869,"":"downstairs","":"downstairs","session_id":"aebeab86-ba4f-4eb8-bba6-a10f5cf0cea7"} {"msg":"[1]R generation[0..12]: [180, 176, 179, 181, 183, 182, 182, 183, 180, 182, 182, 180]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:34.095801876Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1869,"":"downstairs","":"downstairs","session_id":"aebeab86-ba4f-4eb8-bba6-a10f5cf0cea7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:34.095812846Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1869,"":"downstairs","":"downstairs","session_id":"aebeab86-ba4f-4eb8-bba6-a10f5cf0cea7"} {"msg":"[2]R flush_numbers[0..12]: [304, 293, 303, 307, 311, 308, 308, 311, 305, 309, 308, 306]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:34.095823366Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1869,"":"downstairs","":"downstairs","session_id":"aebeab86-ba4f-4eb8-bba6-a10f5cf0cea7"} {"msg":"[2]R generation[0..12]: [180, 176, 179, 181, 183, 182, 182, 183, 180, 182, 182, 180]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:34.095833855Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1869,"":"downstairs","":"downstairs","session_id":"aebeab86-ba4f-4eb8-bba6-a10f5cf0cea7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:34.095846865Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1869,"":"downstairs","":"downstairs","session_id":"aebeab86-ba4f-4eb8-bba6-a10f5cf0cea7"} {"msg":"Max found gen is 184","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:34.095856245Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1869,"":"downstairs","":"downstairs","session_id":"aebeab86-ba4f-4eb8-bba6-a10f5cf0cea7"} {"msg":"Generation requested: 184 >= found:184","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:34.095865944Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1869,"":"downstairs","":"downstairs","session_id":"aebeab86-ba4f-4eb8-bba6-a10f5cf0cea7"} {"msg":"Next flush: 312","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:34.095875234Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1869,"":"downstairs","":"downstairs","session_id":"aebeab86-ba4f-4eb8-bba6-a10f5cf0cea7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:34.095884624Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1869,"":"downstairs","":"downstairs","session_id":"aebeab86-ba4f-4eb8-bba6-a10f5cf0cea7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:34.095894244Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1869,"session_id":"aebeab86-ba4f-4eb8-bba6-a10f5cf0cea7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:34.095902793Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1869,"client":"0","":"downstairs","session_id":"aebeab86-ba4f-4eb8-bba6-a10f5cf0cea7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:34.095912453Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1869,"client":"1","":"downstairs","session_id":"aebeab86-ba4f-4eb8-bba6-a10f5cf0cea7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:34.095922063Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1869,"client":"2","":"downstairs","session_id":"aebeab86-ba4f-4eb8-bba6-a10f5cf0cea7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:34.095931832Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1869,"session_id":"aebeab86-ba4f-4eb8-bba6-a10f5cf0cea7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:34.095940882Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1869,"session_id":"aebeab86-ba4f-4eb8-bba6-a10f5cf0cea7"} {"msg":"21ed9e98-0739-4f31-b34e-8fc0f4775edc is now active with session: aebeab86-ba4f-4eb8-bba6-a10f5cf0cea7","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:34.095950672Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1869,"session_id":"aebeab86-ba4f-4eb8-bba6-a10f5cf0cea7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:34.095959392Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1869,"session_id":"aebeab86-ba4f-4eb8-bba6-a10f5cf0cea7"} {"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:34.095968511Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1869} {"msg":"Activated sub_volume 21ed9e98-0739-4f31-b34e-8fc0f4775edc","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:34.095976491Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1869} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:34.095984501Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1869,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:34.096300302Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1869,"task":"crutest"} 01/30 Flush 02/30 Write block 105 len 8192 data: 7 7 03/30 Read block 64 len 8192 04/30 Write block 587 len 16384 data: 22 19 12 12 05/30 Read block 498 len 16384 06/30 Flush 07/30 Write block 27 len 4096 data: 9 08/30 Write block 243 len 28672 data: 15 15 15 17 18 17 14 09/30 Read block 556 len 12288 10/30 Read block 379 len 24576 11/30 Read block 279 len 32768 12/30 Read block 40 len 20480 13/30 Read block 515 len 20480 14/30 Read block 528 len 32768 15/30 Write block 432 len 4096 data: 12 16/30 Write block 122 len 32768 data: 15 17 15 15 14 14 15 13 17/30 Write block 166 len 32768 data: 14 13 12 13 15 15 18 19 18/30 Read block 349 len 12288 19/30 Read block 4 len 32768 20/30 Read block 592 len 16384 21/30 Read block 127 len 28672 22/30 Write block 317 len 16384 data: 14 9 12 11 23/30 Write block 87 len 8192 data: 12 11 24/30 Write block 189 len 36864 data: 15 15 14 14 14 16 13 15 15 25/30 Read block 378 len 40960 26/30 Flush 27/30 Write block 179 len 24576 data: 16 19 16 15 14 16 28/30 Read block 341 len 40960 29/30 Read block 98 len 24576 30/30 Read block 132 len 20480 ---------------------------------------------------------------- Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 2 Done Done Done false 1002 Acked Read 2 Done Done Sent false 1003 Acked Write 4 Done Done Sent false 1004 Acked Read 4 Done Done Sent false 1005 Acked Flush 0 Done Done Sent false 1006 Acked Write 1 Done Done Sent false 1007 Acked Write 7 Done Done Sent false 1008 Acked Read 3 Done Done Sent false 1009 Acked Read 6 Done Done Sent false 1010 Acked Read 8 Done Done Sent false 1011 Acked Read 5 Done Done Sent false 1012 Acked Read 5 Done Done Sent false 1013 Acked Read 8 Done Done Sent false 1014 Acked Write 1 Done Done Sent false 1015 Acked Write 8 Done Done Sent false 1016 Acked Write 8 Done Done Sent false 1017 Acked Read 3 Done Done Sent false 1018 Acked Read 8 Done Done Sent false 1019 Acked Read 4 Done Done Sent false 1020 Acked Read 7 Done Done Sent false 1021 Acked Write 4 Done Done Sent false 1022 Acked Write 2 Done Done Sent false 1023 Acked Write 9 Done Done Sent false 1024 Acked Read 10 Done Done Sent false 1025 Acked Flush 0 Done Done Sent false 1026 Acked Write 6 Done Done Sent false 1027 Acked Read 10 Done Done Sent false 1028 Acked Read 6 Done Done Sent false 1029 Acked Read 5 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 28 28 Done 29 29 1 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1000)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:48:34.105642872Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1869} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:34.105831597Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1869,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 180 180 180 304 304 304 F F F 1 020-039 184 184 176 314 314 293 F F F <--- 2 040-059 179 179 179 303 303 303 F F F 3 060-079 181 181 181 307 307 307 F F F 4 080-099 184 184 183 314 314 311 F F F <--- 5 100-119 184 184 182 313 313 308 F F T <--- 6 120-139 184 184 182 314 314 308 F F F <--- 7 140-159 183 183 183 311 311 311 F F F 8 160-179 184 184 180 314 314 305 T T F <--- 9 180-199 184 184 182 314 314 309 T T F <--- 10 200-219 182 182 182 308 308 308 F F F 11 220-239 180 180 180 306 306 306 F F F 12 240-259 184 184 182 314 314 309 F F F <--- 13 260-279 181 181 181 307 307 307 F F F 14 280-299 178 178 178 302 302 302 F F F 15 300-319 184 184 182 314 314 309 F F F <--- 16 320-339 184 184 176 314 314 293 F F F <--- 17 340-359 180 180 180 305 305 305 F F F 18 360-379 183 183 183 311 311 311 F F F 19 380-399 183 183 183 311 311 311 F F F 20 400-419 182 182 182 310 310 310 F F F 21 420-439 184 184 181 314 314 307 F F F <--- 22 440-459 179 179 179 303 303 303 F F F 23 460-479 178 178 178 300 300 300 F F F 24 480-499 183 183 183 311 311 311 F F F 25 500-519 183 183 183 311 311 311 F F F 26 520-539 181 181 181 307 307 307 F F F 27 540-559 181 181 181 307 307 307 F F F 28 560-579 180 180 180 306 306 306 F F F 29 580-599 184 184 182 313 313 309 F F F <--- Max gen: 184, Max flush: 314 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 92 Verifying data now No change after verify Loop: 92 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 180 180 180 304 304 304 F F F 1 020-039 184 184 184 314 314 314 F F F 2 040-059 179 179 179 303 303 303 F F F 3 060-079 181 181 181 307 307 307 F F F 4 080-099 184 184 184 314 314 314 F F F 5 100-119 184 184 184 313 313 313 F F F 6 120-139 184 184 184 314 314 314 F F F 7 140-159 183 183 183 311 311 311 F F F 8 160-179 185 185 185 315 315 315 F F F 9 180-199 185 185 185 315 315 315 F F F 10 200-219 182 182 182 308 308 308 F F F 11 220-239 180 180 180 306 306 306 F F F 12 240-259 184 184 184 314 314 314 F F F 13 260-279 181 181 181 307 307 307 F F F 14 280-299 178 178 178 302 302 302 F F F 15 300-319 184 184 184 314 314 314 F F F 16 320-339 184 184 184 314 314 314 F F F 17 340-359 180 180 180 305 305 305 F F F 18 360-379 183 183 183 311 311 311 F F F 19 380-399 183 183 183 311 311 311 F F F 20 400-419 182 182 182 310 310 310 F F F 21 420-439 184 184 184 314 314 314 F F F 22 440-459 179 179 179 303 303 303 F F F 23 460-479 178 178 178 300 300 300 F F F 24 480-499 183 183 183 311 311 311 F F F 25 500-519 183 183 183 311 311 311 F F F 26 520-539 181 181 181 307 307 307 F F F 27 540-559 181 181 181 307 307 307 F F F 28 560-579 180 180 180 306 306 306 F F F 29 580-599 184 184 184 313 313 313 F F F Max gen: 185, Max flush: 315 Begin loop 93 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:38.347375741Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1877,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:38.355316832Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1877,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:38.355595894Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1877} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:38.355624453Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1877} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:38.355632973Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1877} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:38.355662322Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1877} {"msg":"Crucible 9ff3dbf6-34eb-4018-bbb4-b0ef55bdfa34 has session id: 54d2c008-0f68-4fcc-8447-40bf4962516a","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:38.355977893Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1877,"session_id":"54d2c008-0f68-4fcc-8447-40bf4962516a"} {"msg":"Upstairs opts: Upstairs UUID: 9ff3dbf6-34eb-4018-bbb4-b0ef55bdfa34, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:38.356026592Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1877,"session_id":"54d2c008-0f68-4fcc-8447-40bf4962516a"} {"msg":"Crucible stats registered with UUID: 9ff3dbf6-34eb-4018-bbb4-b0ef55bdfa34","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:38.356037541Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1877,"session_id":"54d2c008-0f68-4fcc-8447-40bf4962516a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:48:38.356046771Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1877,"session_id":"54d2c008-0f68-4fcc-8447-40bf4962516a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:38.356188657Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1877,"":"io task","client":"1","":"downstairs","session_id":"54d2c008-0f68-4fcc-8447-40bf4962516a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:38.356221756Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1877,"":"io task","client":"0","":"downstairs","session_id":"54d2c008-0f68-4fcc-8447-40bf4962516a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:38.356232826Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1877,"":"io task","client":"2","":"downstairs","session_id":"54d2c008-0f68-4fcc-8447-40bf4962516a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:38.356333203Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1877,"task":"crutest"} {"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:38.356350072Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1877} {"msg":"9ff3dbf6-34eb-4018-bbb4-b0ef55bdfa34 active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:38.356379231Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1877,"session_id":"54d2c008-0f68-4fcc-8447-40bf4962516a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:38.356388451Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1877,"client":"0","":"downstairs","session_id":"54d2c008-0f68-4fcc-8447-40bf4962516a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:38.356398021Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1877,"client":"1","":"downstairs","session_id":"54d2c008-0f68-4fcc-8447-40bf4962516a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:38.356407601Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1877,"client":"2","":"downstairs","session_id":"54d2c008-0f68-4fcc-8447-40bf4962516a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:38.3564171Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1877,"":"io task","client":"2","":"downstairs","session_id":"54d2c008-0f68-4fcc-8447-40bf4962516a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:38.35642757Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1877,"":"io task","client":"0","":"downstairs","session_id":"54d2c008-0f68-4fcc-8447-40bf4962516a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:38.35643793Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1877,"":"io task","client":"1","":"downstairs","session_id":"54d2c008-0f68-4fcc-8447-40bf4962516a"} {"msg":"ds_connection connected from Ok(127.0.0.1:53524)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:38.356479919Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1877,"":"io task","client":"1","":"downstairs","session_id":"54d2c008-0f68-4fcc-8447-40bf4962516a"} {"msg":"ds_connection connected from Ok(127.0.0.1:57911)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:38.356490728Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1877,"":"io task","client":"2","":"downstairs","session_id":"54d2c008-0f68-4fcc-8447-40bf4962516a"} {"msg":"ds_connection connected from Ok(127.0.0.1:47424)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:38.356522677Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1877,"":"io task","client":"0","":"downstairs","session_id":"54d2c008-0f68-4fcc-8447-40bf4962516a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:38.357102381Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1877,"client":"0","":"downstairs","session_id":"54d2c008-0f68-4fcc-8447-40bf4962516a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:38.357174649Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1877,"client":"2","":"downstairs","session_id":"54d2c008-0f68-4fcc-8447-40bf4962516a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:38.357228507Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1877,"client":"1","":"downstairs","session_id":"54d2c008-0f68-4fcc-8447-40bf4962516a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:38.357380503Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1877,"session_id":"54d2c008-0f68-4fcc-8447-40bf4962516a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:38.357423901Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1877,"session_id":"54d2c008-0f68-4fcc-8447-40bf4962516a"} {"msg":"[0]R flush_numbers[0..12]: [304, 314, 303, 307, 314, 313, 314, 311, 315, 315, 308, 306]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:38.357509259Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1877,"":"downstairs","":"downstairs","session_id":"54d2c008-0f68-4fcc-8447-40bf4962516a"} {"msg":"[0]R generation[0..12]: [180, 184, 179, 181, 184, 184, 184, 183, 185, 185, 182, 180]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:38.357520489Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1877,"":"downstairs","":"downstairs","session_id":"54d2c008-0f68-4fcc-8447-40bf4962516a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:38.357531628Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1877,"":"downstairs","":"downstairs","session_id":"54d2c008-0f68-4fcc-8447-40bf4962516a"} {"msg":"[1]R flush_numbers[0..12]: [304, 314, 303, 307, 314, 313, 314, 311, 315, 315, 308, 306]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:38.357542168Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1877,"":"downstairs","":"downstairs","session_id":"54d2c008-0f68-4fcc-8447-40bf4962516a"} {"msg":"[1]R generation[0..12]: [180, 184, 179, 181, 184, 184, 184, 183, 185, 185, 182, 180]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:38.357552758Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1877,"":"downstairs","":"downstairs","session_id":"54d2c008-0f68-4fcc-8447-40bf4962516a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:38.357563717Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1877,"":"downstairs","":"downstairs","session_id":"54d2c008-0f68-4fcc-8447-40bf4962516a"} {"msg":"[2]R flush_numbers[0..12]: [304, 314, 303, 307, 314, 313, 314, 311, 315, 315, 308, 306]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:38.357574497Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1877,"":"downstairs","":"downstairs","session_id":"54d2c008-0f68-4fcc-8447-40bf4962516a"} {"msg":"[2]R generation[0..12]: [180, 184, 179, 181, 184, 184, 184, 183, 185, 185, 182, 180]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:38.357584957Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1877,"":"downstairs","":"downstairs","session_id":"54d2c008-0f68-4fcc-8447-40bf4962516a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:38.357595796Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1877,"":"downstairs","":"downstairs","session_id":"54d2c008-0f68-4fcc-8447-40bf4962516a"} {"msg":"Max found gen is 186","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:38.357605196Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1877,"":"downstairs","":"downstairs","session_id":"54d2c008-0f68-4fcc-8447-40bf4962516a"} {"msg":"Generation requested: 186 >= found:186","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:38.357614926Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1877,"":"downstairs","":"downstairs","session_id":"54d2c008-0f68-4fcc-8447-40bf4962516a"} {"msg":"Next flush: 316","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:38.357624346Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1877,"":"downstairs","":"downstairs","session_id":"54d2c008-0f68-4fcc-8447-40bf4962516a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:38.357633735Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1877,"":"downstairs","":"downstairs","session_id":"54d2c008-0f68-4fcc-8447-40bf4962516a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:38.357645065Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1877,"session_id":"54d2c008-0f68-4fcc-8447-40bf4962516a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:38.357653855Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1877,"client":"0","":"downstairs","session_id":"54d2c008-0f68-4fcc-8447-40bf4962516a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:38.357663404Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1877,"client":"1","":"downstairs","session_id":"54d2c008-0f68-4fcc-8447-40bf4962516a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:38.357673004Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1877,"client":"2","":"downstairs","session_id":"54d2c008-0f68-4fcc-8447-40bf4962516a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:38.357682704Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1877,"session_id":"54d2c008-0f68-4fcc-8447-40bf4962516a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:38.357691834Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1877,"session_id":"54d2c008-0f68-4fcc-8447-40bf4962516a"} {"msg":"9ff3dbf6-34eb-4018-bbb4-b0ef55bdfa34 is now active with session: 54d2c008-0f68-4fcc-8447-40bf4962516a","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:38.357701583Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1877,"session_id":"54d2c008-0f68-4fcc-8447-40bf4962516a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:38.357710243Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1877,"session_id":"54d2c008-0f68-4fcc-8447-40bf4962516a"} {"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:38.357719363Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1877} {"msg":"Activated sub_volume 9ff3dbf6-34eb-4018-bbb4-b0ef55bdfa34","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:38.357727323Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1877} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:38.357735342Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1877,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:38.358044983Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1877,"task":"crutest"} 01/30 Write block 79 len 20480 data: 18 13 12 13 13 02/30 Write block 139 len 20480 data: 13 15 18 20 18 03/30 Read block 345 len 20480 04/30 Read block 217 len 4096 05/30 Write block 302 len 32768 data: 9 10 10 12 11 11 10 13 06/30 Read block 512 len 8192 07/30 Read block 348 len 20480 08/30 Read block 137 len 36864 09/30 Read block 335 len 12288 10/30 Write block 162 len 36864 data: 9 11 8 11 15 14 13 14 16 11/30 Write block 174 len 16384 data: 16 15 14 14 12/30 Read block 504 len 28672 13/30 Write block 131 len 36864 data: 14 13 13 13 12 14 10 10 14 14/30 Write block 399 len 40960 data: 17 19 18 19 16 17 17 14 13 9 15/30 Flush 16/30 Write block 9 len 28672 data: 16 16 13 10 12 12 12 17/30 Read block 414 len 36864 18/30 Read block 355 len 28672 19/30 Read block 577 len 4096 20/30 Read block 47 len 28672 21/30 Write block 289 len 4096 data: 14 22/30 Read block 270 len 32768 23/30 Write block 23 len 20480 data: 10 11 11 8 10 24/30 Write block 256 len 32768 data: 17 14 15 14 14 13 14 12 25/30 Flush 26/30 Write block 12 len 12288 data: 11 13 13 27/30 Read block 189 len 16384 28/30 Write block 480 len 20480 data: 15 18 13 11 12 29/30 Write block 189 len 32768 data: 16 16 15 15 15 17 14 16 30/30 Write block 366 len 32768 data: 12 14 12 12 10 8 11 11 ---------------------------------------------------------------- Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Write 5 Done Done Done false 1002 Acked Read 5 Done Sent Done false 1003 Acked Read 1 Done Sent Done false 1004 Acked Write 8 Done Sent Done false 1005 Acked Read 2 Done Sent Done false 1006 Acked Read 5 Done Sent Done false 1007 Acked Read 9 Done Sent Done false 1008 Acked Read 3 Done Sent Done false 1009 Acked Write 9 Done Sent Done false 1010 Acked Write 4 Done Sent Done false 1011 Acked Read 7 Done Sent Done false 1012 Acked Write 9 Done Sent Done false 1013 Acked Write 10 Done Sent Done false 1014 Acked Flush 0 Done Sent Done false 1015 Acked Write 7 Done Sent Done false 1016 Acked Read 9 Done Sent Done false 1017 Acked Read 7 Done Sent Done false 1018 Acked Read 1 Done Sent Done false 1019 Acked Read 7 Done Sent Done false 1020 Acked Write 1 Done Sent Done false 1021 Acked Read 8 Done Sent Done false 1022 Acked Write 5 Done Sent Done false 1023 Acked Write 8 Done Sent Done false 1024 Acked Flush 0 Done Sent Done false 1025 Acked Write 3 Done Sent Done false 1026 Acked Read 4 Done Sent Done false 1027 Acked Write 5 Done Sent Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 28 2 32 Done 28 2 28 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) None Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:48:38.366427632Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1877} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:38.366608446Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1877,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 186 180 186 317 304 317 T F T <--- 1 020-039 186 184 186 317 314 317 F F F <--- 2 040-059 179 179 179 303 303 303 F F F 3 060-079 186 181 186 316 307 316 F T F <--- 4 080-099 186 184 186 316 314 316 F T F <--- 5 100-119 184 184 184 313 313 313 F F F 6 120-139 186 184 186 316 314 316 F T F <--- 7 140-159 186 183 186 316 311 316 F T F <--- 8 160-179 186 185 186 316 315 316 F F F <--- 9 180-199 185 185 185 315 315 315 T F T <--- 10 200-219 182 182 182 308 308 308 F F F 11 220-239 180 180 180 306 306 306 F F F 12 240-259 186 184 186 317 314 317 F F F <--- 13 260-279 186 181 186 317 307 317 F F F <--- 14 280-299 186 178 186 317 302 317 F F F <--- 15 300-319 186 184 186 316 314 316 F F F <--- 16 320-339 184 184 184 314 314 314 F F F 17 340-359 180 180 180 305 305 305 F F F 18 360-379 183 183 183 311 311 311 T F T <--- 19 380-399 186 183 186 316 311 316 F F F <--- 20 400-419 186 182 186 316 310 316 F F F <--- 21 420-439 184 184 184 314 314 314 F F F 22 440-459 179 179 179 303 303 303 F F F 23 460-479 178 178 178 300 300 300 F F F 24 480-499 183 183 183 311 311 311 T F T <--- 25 500-519 183 183 183 311 311 311 F F F 26 520-539 181 181 181 307 307 307 F F F 27 540-559 181 181 181 307 307 307 F F F 28 560-579 180 180 180 306 306 306 F F F 29 580-599 184 184 184 313 313 313 F F F Max gen: 186, Max flush: 317 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 93 Verifying data now No change after verify Loop: 93 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 187 187 187 318 318 318 F F F 1 020-039 186 186 186 317 317 317 F F F 2 040-059 179 179 179 303 303 303 F F F 3 060-079 186 186 186 316 316 316 F F F 4 080-099 186 186 186 316 316 316 F F F 5 100-119 184 184 184 313 313 313 F F F 6 120-139 186 186 186 316 316 316 F F F 7 140-159 186 186 186 316 316 316 F F F 8 160-179 186 186 186 316 316 316 F F F 9 180-199 187 187 187 318 318 318 F F F 10 200-219 182 182 182 308 308 308 F F F 11 220-239 180 180 180 306 306 306 F F F 12 240-259 186 186 186 317 317 317 F F F 13 260-279 186 186 186 317 317 317 F F F 14 280-299 186 186 186 317 317 317 F F F 15 300-319 186 186 186 316 316 316 F F F 16 320-339 184 184 184 314 314 314 F F F 17 340-359 180 180 180 305 305 305 F F F 18 360-379 187 187 187 318 318 318 F F F 19 380-399 186 186 186 316 316 316 F F F 20 400-419 186 186 186 316 316 316 F F F 21 420-439 184 184 184 314 314 314 F F F 22 440-459 179 179 179 303 303 303 F F F 23 460-479 178 178 178 300 300 300 F F F 24 480-499 187 187 187 318 318 318 F F F 25 500-519 183 183 183 311 311 311 F F F 26 520-539 181 181 181 307 307 307 F F F 27 540-559 181 181 181 307 307 307 F F F 28 560-579 180 180 180 306 306 306 F F F 29 580-599 184 184 184 313 313 313 F F F Max gen: 187, Max flush: 318 Begin loop 94 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:42.621844936Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1885,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:48:42.628852354Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1885,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:42.628905532Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1885,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:42.635675337Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1885,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:42.635935189Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1885} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:42.635960419Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1885} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:42.635969098Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1885} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:42.635980628Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1885} {"msg":"Crucible a622d560-e96b-4d5e-84d2-21604b9a0b9d has session id: 0a535da1-83c8-4084-a357-2d0b3311d462","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:42.636305099Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1885,"session_id":"0a535da1-83c8-4084-a357-2d0b3311d462"} {"msg":"Upstairs opts: Upstairs UUID: a622d560-e96b-4d5e-84d2-21604b9a0b9d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:42.636334038Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1885,"session_id":"0a535da1-83c8-4084-a357-2d0b3311d462"} {"msg":"Crucible stats registered with UUID: a622d560-e96b-4d5e-84d2-21604b9a0b9d","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:42.636344757Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1885,"session_id":"0a535da1-83c8-4084-a357-2d0b3311d462"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:48:42.636353877Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1885,"session_id":"0a535da1-83c8-4084-a357-2d0b3311d462"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:42.636483663Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1885,"":"io task","client":"0","":"downstairs","session_id":"0a535da1-83c8-4084-a357-2d0b3311d462"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:42.636503193Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1885,"":"io task","client":"2","":"downstairs","session_id":"0a535da1-83c8-4084-a357-2d0b3311d462"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:42.636513403Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1885,"":"io task","client":"1","":"downstairs","session_id":"0a535da1-83c8-4084-a357-2d0b3311d462"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:42.636664298Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1885,"task":"crutest"} {"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:42.636679698Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1885} {"msg":"a622d560-e96b-4d5e-84d2-21604b9a0b9d active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:42.636695417Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1885,"session_id":"0a535da1-83c8-4084-a357-2d0b3311d462"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:42.636704387Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1885,"client":"0","":"downstairs","session_id":"0a535da1-83c8-4084-a357-2d0b3311d462"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:42.636715587Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1885,"client":"1","":"downstairs","session_id":"0a535da1-83c8-4084-a357-2d0b3311d462"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:42.636725186Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1885,"client":"2","":"downstairs","session_id":"0a535da1-83c8-4084-a357-2d0b3311d462"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:42.636734716Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1885,"":"io task","client":"2","":"downstairs","session_id":"0a535da1-83c8-4084-a357-2d0b3311d462"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:42.636744766Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1885,"":"io task","client":"0","":"downstairs","session_id":"0a535da1-83c8-4084-a357-2d0b3311d462"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:42.636754656Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1885,"":"io task","client":"1","":"downstairs","session_id":"0a535da1-83c8-4084-a357-2d0b3311d462"} {"msg":"ds_connection connected from Ok(127.0.0.1:45120)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:42.636805194Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1885,"":"io task","client":"0","":"downstairs","session_id":"0a535da1-83c8-4084-a357-2d0b3311d462"} {"msg":"ds_connection connected from Ok(127.0.0.1:49521)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:42.636815954Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1885,"":"io task","client":"2","":"downstairs","session_id":"0a535da1-83c8-4084-a357-2d0b3311d462"} {"msg":"ds_connection connected from Ok(127.0.0.1:40594)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:42.636901661Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1885,"":"io task","client":"1","":"downstairs","session_id":"0a535da1-83c8-4084-a357-2d0b3311d462"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:42.637550873Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1885,"client":"1","":"downstairs","session_id":"0a535da1-83c8-4084-a357-2d0b3311d462"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:42.637603381Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1885,"client":"0","":"downstairs","session_id":"0a535da1-83c8-4084-a357-2d0b3311d462"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:42.63762796Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1885,"client":"2","":"downstairs","session_id":"0a535da1-83c8-4084-a357-2d0b3311d462"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:42.637886223Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1885,"session_id":"0a535da1-83c8-4084-a357-2d0b3311d462"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:42.637896693Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1885,"session_id":"0a535da1-83c8-4084-a357-2d0b3311d462"} {"msg":"[0]R flush_numbers[0..12]: [318, 317, 303, 316, 316, 313, 316, 316, 316, 318, 308, 306]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:42.637939041Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1885,"":"downstairs","":"downstairs","session_id":"0a535da1-83c8-4084-a357-2d0b3311d462"} {"msg":"[0]R generation[0..12]: [187, 186, 179, 186, 186, 184, 186, 186, 186, 187, 182, 180]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:42.637950231Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1885,"":"downstairs","":"downstairs","session_id":"0a535da1-83c8-4084-a357-2d0b3311d462"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:42.637964111Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1885,"":"downstairs","":"downstairs","session_id":"0a535da1-83c8-4084-a357-2d0b3311d462"} {"msg":"[1]R flush_numbers[0..12]: [318, 317, 303, 316, 316, 313, 316, 316, 316, 318, 308, 306]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:42.63797479Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1885,"":"downstairs","":"downstairs","session_id":"0a535da1-83c8-4084-a357-2d0b3311d462"} {"msg":"[1]R generation[0..12]: [187, 186, 179, 186, 186, 184, 186, 186, 186, 187, 182, 180]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:42.63798515Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1885,"":"downstairs","":"downstairs","session_id":"0a535da1-83c8-4084-a357-2d0b3311d462"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:42.6379959Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1885,"":"downstairs","":"downstairs","session_id":"0a535da1-83c8-4084-a357-2d0b3311d462"} {"msg":"[2]R flush_numbers[0..12]: [318, 317, 303, 316, 316, 313, 316, 316, 316, 318, 308, 306]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:42.638006659Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1885,"":"downstairs","":"downstairs","session_id":"0a535da1-83c8-4084-a357-2d0b3311d462"} {"msg":"[2]R generation[0..12]: [187, 186, 179, 186, 186, 184, 186, 186, 186, 187, 182, 180]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:42.638017279Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1885,"":"downstairs","":"downstairs","session_id":"0a535da1-83c8-4084-a357-2d0b3311d462"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:42.638028049Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1885,"":"downstairs","":"downstairs","session_id":"0a535da1-83c8-4084-a357-2d0b3311d462"} {"msg":"Max found gen is 188","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:42.638037279Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1885,"":"downstairs","":"downstairs","session_id":"0a535da1-83c8-4084-a357-2d0b3311d462"} {"msg":"Generation requested: 188 >= found:188","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:42.638047068Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1885,"":"downstairs","":"downstairs","session_id":"0a535da1-83c8-4084-a357-2d0b3311d462"} {"msg":"Next flush: 319","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:42.638056448Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1885,"":"downstairs","":"downstairs","session_id":"0a535da1-83c8-4084-a357-2d0b3311d462"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:42.638065788Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1885,"":"downstairs","":"downstairs","session_id":"0a535da1-83c8-4084-a357-2d0b3311d462"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:42.638075368Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1885,"session_id":"0a535da1-83c8-4084-a357-2d0b3311d462"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:42.638084097Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1885,"client":"0","":"downstairs","session_id":"0a535da1-83c8-4084-a357-2d0b3311d462"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:42.638093737Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1885,"client":"1","":"downstairs","session_id":"0a535da1-83c8-4084-a357-2d0b3311d462"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:42.638104517Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1885,"client":"2","":"downstairs","session_id":"0a535da1-83c8-4084-a357-2d0b3311d462"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:42.638114326Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1885,"session_id":"0a535da1-83c8-4084-a357-2d0b3311d462"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:42.638123246Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1885,"session_id":"0a535da1-83c8-4084-a357-2d0b3311d462"} {"msg":"a622d560-e96b-4d5e-84d2-21604b9a0b9d is now active with session: 0a535da1-83c8-4084-a357-2d0b3311d462","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:42.638132976Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1885,"session_id":"0a535da1-83c8-4084-a357-2d0b3311d462"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:42.638141576Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1885,"session_id":"0a535da1-83c8-4084-a357-2d0b3311d462"} {"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:42.638150715Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1885} {"msg":"Activated sub_volume a622d560-e96b-4d5e-84d2-21604b9a0b9d","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:42.638158665Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1885} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:42.638166775Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1885,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:42.638465066Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1885,"task":"crutest"} 01/30 Flush 02/30 Write block 406 len 32768 data: 15 14 10 11 14 17 16 16 03/30 Read block 27 len 32768 04/30 Write block 76 len 12288 data: 15 18 20 05/30 Write block 38 len 20480 data: 10 8 10 12 9 06/30 Read block 583 len 28672 07/30 Write block 347 len 8192 data: 12 12 08/30 Read block 508 len 28672 09/30 Write block 187 len 24576 data: 14 14 17 17 16 16 10/30 Read block 582 len 28672 11/30 Read block 595 len 8192 12/30 Write block 60 len 36864 data: 14 16 16 12 10 9 8 9 7 13/30 Flush 14/30 Write block 176 len 32768 data: 15 15 14 17 20 17 16 15 15/30 Write block 206 len 36864 data: 16 12 14 15 15 15 16 19 17 16/30 Read block 389 len 32768 17/30 Read block 551 len 20480 18/30 Read block 249 len 20480 19/30 Flush 20/30 Write block 553 len 4096 data: 15 21/30 Write block 427 len 20480 data: 14 13 17 13 12 22/30 Flush 23/30 Write block 130 len 40960 data: 14 15 14 14 14 13 15 11 11 15 24/30 Read block 267 len 28672 25/30 Flush 26/30 Write block 429 len 40960 data: 18 14 13 13 9 7 11 12 8 9 27/30 Write block 557 len 4096 data: 12 28/30 Write block 363 len 24576 data: 21 21 14 13 15 13 29/30 Write block 22 len 20480 data: 13 11 12 12 9 30/30 Read block 415 len 16384 ---------------------------------------------------------------- Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 8 Sent Done Done false 1002 Acked Read 8 Sent Done Done false 1003 Acked Write 3 Sent Done Done false 1004 Acked Write 5 Sent Done Done false 1005 Acked Read 7 Sent Done Done false 1006 Acked Write 2 Sent Done Done false 1007 Acked Read 7 Sent Done Done false 1008 Acked Write 6 Sent Done Done false 1009 Acked Read 7 Sent Done Done false 1010 Acked Read 2 Sent Done Done false 1011 Acked Write 9 Sent Done Done false 1012 Acked Flush 0 Sent Done Done false 1013 Acked Write 8 Sent Done Done false 1014 Acked Write 9 Sent Done Done false 1015 Acked Read 8 Sent Done Done false 1016 Acked Read 5 Sent Done Done false 1017 Acked Read 5 Sent Done Done false 1018 Acked Flush 0 Sent Done Done false 1019 Acked Write 1 Sent Done Done false 1020 Acked Write 5 Sent Done Done false 1021 Acked Flush 0 Sent Done Done false 1022 Acked Write 10 Sent Done Done false 1023 Acked Read 7 Sent Done Done false 1024 Acked Flush 0 Sent Done Done false 1025 Acked Write 10 Sent Done Done false 1026 Acked Write 1 Sent Done Done false 1027 Acked Write 6 Sent Done Sent false 1028 Acked Write 5 Sent Done Sent false 1029 Acked Read 4 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 0 3 32 Done 0 29 26 55 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1000)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:48:42.647661191Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1885} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:42.647849956Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1885,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 187 187 187 318 318 318 F F F 1 020-039 186 188 188 317 320 320 F T T <--- 2 040-059 179 188 188 303 320 320 F F F <--- 3 060-079 186 188 188 316 320 320 F F F <--- 4 080-099 186 186 186 316 316 316 F F F 5 100-119 184 184 184 313 313 313 F F F 6 120-139 186 188 188 316 323 323 F F F <--- 7 140-159 186 186 186 316 316 316 F F F 8 160-179 186 188 188 316 321 321 F F F <--- 9 180-199 187 188 188 318 321 321 F F F <--- 10 200-219 182 188 188 308 321 321 F F F <--- 11 220-239 180 180 180 306 306 306 F F F 12 240-259 186 186 186 317 317 317 F F F 13 260-279 186 186 186 317 317 317 F F F 14 280-299 186 186 186 317 317 317 F F F 15 300-319 186 186 186 316 316 316 F F F 16 320-339 184 184 184 314 314 314 F F F 17 340-359 180 188 188 305 320 320 F F F <--- 18 360-379 187 187 187 318 318 318 F T T <--- 19 380-399 186 186 186 316 316 316 F F F 20 400-419 186 188 188 316 320 320 F F F <--- 21 420-439 184 188 188 314 322 322 F T T <--- 22 440-459 179 179 179 303 303 303 F F F 23 460-479 178 178 178 300 300 300 F F F 24 480-499 187 187 187 318 318 318 F F F 25 500-519 183 183 183 311 311 311 F F F 26 520-539 181 181 181 307 307 307 F F F 27 540-559 181 188 188 307 322 322 F T T <--- 28 560-579 180 180 180 306 306 306 F F F 29 580-599 184 184 184 313 313 313 F F F Max gen: 188, Max flush: 323 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 94 Verifying data now No change after verify Loop: 94 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 187 187 187 318 318 318 F F F 1 020-039 189 189 189 324 324 324 F F F 2 040-059 188 188 188 320 320 320 F F F 3 060-079 188 188 188 320 320 320 F F F 4 080-099 186 186 186 316 316 316 F F F 5 100-119 184 184 184 313 313 313 F F F 6 120-139 188 188 188 323 323 323 F F F 7 140-159 186 186 186 316 316 316 F F F 8 160-179 188 188 188 321 321 321 F F F 9 180-199 188 188 188 321 321 321 F F F 10 200-219 188 188 188 321 321 321 F F F 11 220-239 180 180 180 306 306 306 F F F 12 240-259 186 186 186 317 317 317 F F F 13 260-279 186 186 186 317 317 317 F F F 14 280-299 186 186 186 317 317 317 F F F 15 300-319 186 186 186 316 316 316 F F F 16 320-339 184 184 184 314 314 314 F F F 17 340-359 188 188 188 320 320 320 F F F 18 360-379 189 189 189 324 324 324 F F F 19 380-399 186 186 186 316 316 316 F F F 20 400-419 188 188 188 320 320 320 F F F 21 420-439 189 189 189 324 324 324 F F F 22 440-459 179 179 179 303 303 303 F F F 23 460-479 178 178 178 300 300 300 F F F 24 480-499 187 187 187 318 318 318 F F F 25 500-519 183 183 183 311 311 311 F F F 26 520-539 181 181 181 307 307 307 F F F 27 540-559 189 189 189 324 324 324 F F F 28 560-579 180 180 180 306 306 306 F F F 29 580-599 184 184 184 313 313 313 F F F Max gen: 189, Max flush: 324 Begin loop 95 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:46.899330143Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1893,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:46.907572715Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1893,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:46.907838547Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1893} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:46.907867576Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1893} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:46.907876176Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1893} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:46.907887486Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1893} {"msg":"Crucible 8e0e766d-97ba-479a-937f-a2045d86be44 has session id: 0fde2f19-d644-4982-b24e-7c3618bd19e6","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:46.908204427Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1893,"session_id":"0fde2f19-d644-4982-b24e-7c3618bd19e6"} {"msg":"Upstairs opts: Upstairs UUID: 8e0e766d-97ba-479a-937f-a2045d86be44, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:46.908230536Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1893,"session_id":"0fde2f19-d644-4982-b24e-7c3618bd19e6"} {"msg":"Crucible stats registered with UUID: 8e0e766d-97ba-479a-937f-a2045d86be44","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:46.908240576Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1893,"session_id":"0fde2f19-d644-4982-b24e-7c3618bd19e6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:48:46.908249425Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1893,"session_id":"0fde2f19-d644-4982-b24e-7c3618bd19e6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:46.908396531Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1893,"":"io task","client":"2","":"downstairs","session_id":"0fde2f19-d644-4982-b24e-7c3618bd19e6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:46.90842743Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1893,"":"io task","client":"0","":"downstairs","session_id":"0fde2f19-d644-4982-b24e-7c3618bd19e6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:46.90843817Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1893,"":"io task","client":"1","":"downstairs","session_id":"0fde2f19-d644-4982-b24e-7c3618bd19e6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:46.908578936Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1893,"task":"crutest"} {"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:46.908601925Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1893} {"msg":"8e0e766d-97ba-479a-937f-a2045d86be44 active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:46.908610705Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1893,"session_id":"0fde2f19-d644-4982-b24e-7c3618bd19e6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:46.908619655Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1893,"client":"0","":"downstairs","session_id":"0fde2f19-d644-4982-b24e-7c3618bd19e6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:46.908629834Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1893,"client":"1","":"downstairs","session_id":"0fde2f19-d644-4982-b24e-7c3618bd19e6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:46.908639314Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1893,"client":"2","":"downstairs","session_id":"0fde2f19-d644-4982-b24e-7c3618bd19e6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:46.908648904Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1893,"":"io task","client":"2","":"downstairs","session_id":"0fde2f19-d644-4982-b24e-7c3618bd19e6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:46.908659004Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1893,"":"io task","client":"0","":"downstairs","session_id":"0fde2f19-d644-4982-b24e-7c3618bd19e6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:46.908669123Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1893,"":"io task","client":"1","":"downstairs","session_id":"0fde2f19-d644-4982-b24e-7c3618bd19e6"} {"msg":"ds_connection connected from Ok(127.0.0.1:63780)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:46.908739521Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1893,"":"io task","client":"1","":"downstairs","session_id":"0fde2f19-d644-4982-b24e-7c3618bd19e6"} {"msg":"ds_connection connected from Ok(127.0.0.1:61952)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:46.908750741Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1893,"":"io task","client":"0","":"downstairs","session_id":"0fde2f19-d644-4982-b24e-7c3618bd19e6"} {"msg":"ds_connection connected from Ok(127.0.0.1:55489)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:46.908760871Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1893,"":"io task","client":"2","":"downstairs","session_id":"0fde2f19-d644-4982-b24e-7c3618bd19e6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:46.909432651Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1893,"client":"0","":"downstairs","session_id":"0fde2f19-d644-4982-b24e-7c3618bd19e6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:46.909457011Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1893,"client":"1","":"downstairs","session_id":"0fde2f19-d644-4982-b24e-7c3618bd19e6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:46.909501339Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1893,"client":"2","":"downstairs","session_id":"0fde2f19-d644-4982-b24e-7c3618bd19e6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:46.909750402Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1893,"session_id":"0fde2f19-d644-4982-b24e-7c3618bd19e6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:46.90982621Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1893,"session_id":"0fde2f19-d644-4982-b24e-7c3618bd19e6"} {"msg":"[0]R flush_numbers[0..12]: [318, 324, 320, 320, 316, 313, 323, 316, 321, 321, 321, 306]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:46.909873509Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1893,"":"downstairs","":"downstairs","session_id":"0fde2f19-d644-4982-b24e-7c3618bd19e6"} {"msg":"[0]R generation[0..12]: [187, 189, 188, 188, 186, 184, 188, 186, 188, 188, 188, 180]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:46.909885088Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1893,"":"downstairs","":"downstairs","session_id":"0fde2f19-d644-4982-b24e-7c3618bd19e6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:46.909896228Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1893,"":"downstairs","":"downstairs","session_id":"0fde2f19-d644-4982-b24e-7c3618bd19e6"} {"msg":"[1]R flush_numbers[0..12]: [318, 324, 320, 320, 316, 313, 323, 316, 321, 321, 321, 306]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:46.909906898Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1893,"":"downstairs","":"downstairs","session_id":"0fde2f19-d644-4982-b24e-7c3618bd19e6"} {"msg":"[1]R generation[0..12]: [187, 189, 188, 188, 186, 184, 188, 186, 188, 188, 188, 180]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:46.909917657Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1893,"":"downstairs","":"downstairs","session_id":"0fde2f19-d644-4982-b24e-7c3618bd19e6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:46.909928607Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1893,"":"downstairs","":"downstairs","session_id":"0fde2f19-d644-4982-b24e-7c3618bd19e6"} {"msg":"[2]R flush_numbers[0..12]: [318, 324, 320, 320, 316, 313, 323, 316, 321, 321, 321, 306]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:46.909941487Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1893,"":"downstairs","":"downstairs","session_id":"0fde2f19-d644-4982-b24e-7c3618bd19e6"} {"msg":"[2]R generation[0..12]: [187, 189, 188, 188, 186, 184, 188, 186, 188, 188, 188, 180]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:46.909952036Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1893,"":"downstairs","":"downstairs","session_id":"0fde2f19-d644-4982-b24e-7c3618bd19e6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:46.909962956Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1893,"":"downstairs","":"downstairs","session_id":"0fde2f19-d644-4982-b24e-7c3618bd19e6"} {"msg":"Max found gen is 190","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:46.909972466Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1893,"":"downstairs","":"downstairs","session_id":"0fde2f19-d644-4982-b24e-7c3618bd19e6"} {"msg":"Generation requested: 190 >= found:190","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:46.909982175Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1893,"":"downstairs","":"downstairs","session_id":"0fde2f19-d644-4982-b24e-7c3618bd19e6"} {"msg":"Next flush: 325","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:46.909991555Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1893,"":"downstairs","":"downstairs","session_id":"0fde2f19-d644-4982-b24e-7c3618bd19e6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:46.910000965Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1893,"":"downstairs","":"downstairs","session_id":"0fde2f19-d644-4982-b24e-7c3618bd19e6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:46.910010535Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1893,"session_id":"0fde2f19-d644-4982-b24e-7c3618bd19e6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:46.910019134Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1893,"client":"0","":"downstairs","session_id":"0fde2f19-d644-4982-b24e-7c3618bd19e6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:46.910028894Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1893,"client":"1","":"downstairs","session_id":"0fde2f19-d644-4982-b24e-7c3618bd19e6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:46.910038654Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1893,"client":"2","":"downstairs","session_id":"0fde2f19-d644-4982-b24e-7c3618bd19e6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:46.910048504Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1893,"session_id":"0fde2f19-d644-4982-b24e-7c3618bd19e6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:46.910057543Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1893,"session_id":"0fde2f19-d644-4982-b24e-7c3618bd19e6"} {"msg":"8e0e766d-97ba-479a-937f-a2045d86be44 is now active with session: 0fde2f19-d644-4982-b24e-7c3618bd19e6","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:46.910067513Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1893,"session_id":"0fde2f19-d644-4982-b24e-7c3618bd19e6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:46.910076183Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1893,"session_id":"0fde2f19-d644-4982-b24e-7c3618bd19e6"} {"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:46.910085192Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1893} {"msg":"Activated sub_volume 8e0e766d-97ba-479a-937f-a2045d86be44","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:46.910094572Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1893} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:46.910102642Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1893,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:46.910446022Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1893,"task":"crutest"} 01/30 Flush 02/30 Write block 62 len 12288 data: 17 13 11 03/30 Read block 186 len 16384 04/30 Read block 304 len 24576 05/30 Flush 06/30 Write block 271 len 12288 data: 12 10 12 07/30 Write block 443 len 12288 data: 8 10 9 08/30 Write block 66 len 40960 data: 9 10 8 9 13 13 14 15 17 16 09/30 Write block 402 len 12288 data: 20 17 18 10/30 Flush 11/30 Write block 351 len 8192 data: 6 7 12/30 Flush 13/30 Write block 430 len 28672 data: 15 14 14 10 8 12 13 14/30 Write block 344 len 20480 data: 11 12 13 13 13 15/30 Write block 50 len 8192 data: 13 12 16/30 Read block 325 len 28672 17/30 Write block 159 len 36864 data: 18 15 12 10 12 9 12 16 15 18/30 Read block 362 len 8192 19/30 Read block 430 len 32768 20/30 Write block 263 len 32768 data: 13 9 9 10 10 9 9 9 21/30 Read block 560 len 4096 22/30 Read block 355 len 4096 23/30 Write block 526 len 8192 data: 11 11 24/30 Read block 424 len 40960 25/30 Write block 359 len 8192 data: 14 15 26/30 Read block 568 len 12288 27/30 Write block 110 len 32768 data: 9 12 17 16 14 15 16 14 28/30 Write block 74 len 12288 data: 18 17 16 29/30 Read block 394 len 28672 30/30 Write block 307 len 40960 data: 12 11 14 15 15 15 17 18 18 15 ---------------------------------------------------------------- Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 7 Done Done Done false 1013 Acked Write 5 Done Done Done false 1014 Acked Write 2 Done Done Sent false 1015 Acked Read 7 Done Done Sent false 1016 Acked Write 9 Done Done Sent false 1017 Acked Read 2 Done Done Sent false 1018 Acked Read 8 Done Done Sent false 1019 Acked Write 8 Done Done Sent false 1020 Acked Read 1 Done Done Sent false 1021 Acked Read 1 Done Done Sent false 1022 Acked Write 2 Done Done Sent false 1023 Acked Read 10 Done Done Sent false 1024 Acked Write 2 Done Done Sent false 1025 Acked Read 3 Done Done Sent false 1026 Acked Write 8 Done Done Sent false 1027 Acked Write 3 Done Done Sent false 1028 Acked Read 7 Done Done Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 16 18 Done 17 17 2 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1011)) Some(JobId(1011)) Some(JobId(1011)) Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:48:46.918557548Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1893} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:46.918793601Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1893,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 187 187 187 318 318 318 F F F 1 020-039 189 189 189 324 324 324 F F F 2 040-059 188 188 188 320 320 320 T T F <--- 3 060-079 190 190 190 327 327 327 T T F <--- 4 080-099 186 186 186 316 316 316 F F F 5 100-119 184 184 184 313 313 313 T T F <--- 6 120-139 188 188 188 323 323 323 F F F 7 140-159 186 186 186 316 316 316 T T F <--- 8 160-179 188 188 188 321 321 321 T T F <--- 9 180-199 188 188 188 321 321 321 F F F 10 200-219 188 188 188 321 321 321 F F F 11 220-239 180 180 180 306 306 306 F F F 12 240-259 186 186 186 317 317 317 F F F 13 260-279 190 190 190 327 327 327 T T F <--- 14 280-299 186 186 186 317 317 317 F F F 15 300-319 186 186 186 316 316 316 T T F <--- 16 320-339 184 184 184 314 314 314 F F F 17 340-359 190 190 190 328 328 328 T T T 18 360-379 189 189 189 324 324 324 T T F <--- 19 380-399 186 186 186 316 316 316 F F F 20 400-419 190 190 190 327 327 327 F F F 21 420-439 189 189 189 324 324 324 T T T 22 440-459 190 190 190 327 327 327 F F F 23 460-479 178 178 178 300 300 300 F F F 24 480-499 187 187 187 318 318 318 F F F 25 500-519 183 183 183 311 311 311 F F F 26 520-539 181 181 181 307 307 307 T T F <--- 27 540-559 189 189 189 324 324 324 F F F 28 560-579 180 180 180 306 306 306 F F F 29 580-599 184 184 184 313 313 313 F F F Max gen: 190, Max flush: 328 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 95 Verifying data now No change after verify Loop: 95 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 187 187 187 318 318 318 F F F 1 020-039 189 189 189 324 324 324 F F F 2 040-059 191 191 191 329 329 329 F F F 3 060-079 191 191 191 329 329 329 F F F 4 080-099 186 186 186 316 316 316 F F F 5 100-119 191 191 191 329 329 329 F F F 6 120-139 188 188 188 323 323 323 F F F 7 140-159 191 191 191 329 329 329 F F F 8 160-179 191 191 191 329 329 329 F F F 9 180-199 188 188 188 321 321 321 F F F 10 200-219 188 188 188 321 321 321 F F F 11 220-239 180 180 180 306 306 306 F F F 12 240-259 186 186 186 317 317 317 F F F 13 260-279 191 191 191 329 329 329 F F F 14 280-299 186 186 186 317 317 317 F F F 15 300-319 191 191 191 329 329 329 F F F 16 320-339 184 184 184 314 314 314 F F F 17 340-359 191 191 191 329 329 329 F F F 18 360-379 191 191 191 329 329 329 F F F 19 380-399 186 186 186 316 316 316 F F F 20 400-419 190 190 190 327 327 327 F F F 21 420-439 191 191 191 329 329 329 F F F 22 440-459 190 190 190 327 327 327 F F F 23 460-479 178 178 178 300 300 300 F F F 24 480-499 187 187 187 318 318 318 F F F 25 500-519 183 183 183 311 311 311 F F F 26 520-539 191 191 191 329 329 329 F F F 27 540-559 189 189 189 324 324 324 F F F 28 560-579 180 180 180 306 306 306 F F F 29 580-599 184 184 184 313 313 313 F F F Max gen: 191, Max flush: 329 Begin loop 96 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:51.170817863Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1901,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:51.178808423Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1901,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:51.179101724Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1901} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:51.179130343Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1901} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:51.179139023Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1901} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:51.179150363Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1901} {"msg":"Crucible 3129432b-133b-40eb-aec3-dc733ae2dd03 has session id: 29b3e9ab-71a3-4cb2-b932-94277a421fcc","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:51.179518212Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1901,"session_id":"29b3e9ab-71a3-4cb2-b932-94277a421fcc"} {"msg":"Upstairs opts: Upstairs UUID: 3129432b-133b-40eb-aec3-dc733ae2dd03, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:51.179547081Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1901,"session_id":"29b3e9ab-71a3-4cb2-b932-94277a421fcc"} {"msg":"Crucible stats registered with UUID: 3129432b-133b-40eb-aec3-dc733ae2dd03","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:51.179557971Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1901,"session_id":"29b3e9ab-71a3-4cb2-b932-94277a421fcc"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:48:51.179567141Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1901,"session_id":"29b3e9ab-71a3-4cb2-b932-94277a421fcc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:51.179696477Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1901,"":"io task","client":"2","":"downstairs","session_id":"29b3e9ab-71a3-4cb2-b932-94277a421fcc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:51.179726736Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1901,"":"io task","client":"0","":"downstairs","session_id":"29b3e9ab-71a3-4cb2-b932-94277a421fcc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:51.179737316Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1901,"":"io task","client":"1","":"downstairs","session_id":"29b3e9ab-71a3-4cb2-b932-94277a421fcc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:51.179901141Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1901,"task":"crutest"} {"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:51.179915861Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1901} {"msg":"3129432b-133b-40eb-aec3-dc733ae2dd03 active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:51.17993939Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1901,"session_id":"29b3e9ab-71a3-4cb2-b932-94277a421fcc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:51.17994901Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1901,"client":"0","":"downstairs","session_id":"29b3e9ab-71a3-4cb2-b932-94277a421fcc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:51.17995856Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1901,"client":"1","":"downstairs","session_id":"29b3e9ab-71a3-4cb2-b932-94277a421fcc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:51.179968109Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1901,"client":"2","":"downstairs","session_id":"29b3e9ab-71a3-4cb2-b932-94277a421fcc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:51.179977709Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1901,"":"io task","client":"2","":"downstairs","session_id":"29b3e9ab-71a3-4cb2-b932-94277a421fcc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:51.179987619Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1901,"":"io task","client":"0","":"downstairs","session_id":"29b3e9ab-71a3-4cb2-b932-94277a421fcc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:51.179997428Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1901,"":"io task","client":"1","":"downstairs","session_id":"29b3e9ab-71a3-4cb2-b932-94277a421fcc"} {"msg":"ds_connection connected from Ok(127.0.0.1:57435)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:51.180035967Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1901,"":"io task","client":"2","":"downstairs","session_id":"29b3e9ab-71a3-4cb2-b932-94277a421fcc"} {"msg":"ds_connection connected from Ok(127.0.0.1:54547)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:51.180047437Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1901,"":"io task","client":"1","":"downstairs","session_id":"29b3e9ab-71a3-4cb2-b932-94277a421fcc"} {"msg":"ds_connection connected from Ok(127.0.0.1:34683)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:51.180057747Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1901,"":"io task","client":"0","":"downstairs","session_id":"29b3e9ab-71a3-4cb2-b932-94277a421fcc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:51.180691418Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1901,"client":"0","":"downstairs","session_id":"29b3e9ab-71a3-4cb2-b932-94277a421fcc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:51.180791096Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1901,"client":"2","":"downstairs","session_id":"29b3e9ab-71a3-4cb2-b932-94277a421fcc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:51.180809645Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1901,"client":"1","":"downstairs","session_id":"29b3e9ab-71a3-4cb2-b932-94277a421fcc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:51.18099139Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1901,"session_id":"29b3e9ab-71a3-4cb2-b932-94277a421fcc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:51.181048558Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1901,"session_id":"29b3e9ab-71a3-4cb2-b932-94277a421fcc"} {"msg":"[0]R flush_numbers[0..12]: [318, 324, 329, 329, 316, 329, 323, 329, 329, 321, 321, 306]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:51.181101097Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1901,"":"downstairs","":"downstairs","session_id":"29b3e9ab-71a3-4cb2-b932-94277a421fcc"} {"msg":"[0]R generation[0..12]: [187, 189, 191, 191, 186, 191, 188, 191, 191, 188, 188, 180]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:51.181112286Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1901,"":"downstairs","":"downstairs","session_id":"29b3e9ab-71a3-4cb2-b932-94277a421fcc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:51.181123326Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1901,"":"downstairs","":"downstairs","session_id":"29b3e9ab-71a3-4cb2-b932-94277a421fcc"} {"msg":"[1]R flush_numbers[0..12]: [318, 324, 329, 329, 316, 329, 323, 329, 329, 321, 321, 306]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:51.181133936Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1901,"":"downstairs","":"downstairs","session_id":"29b3e9ab-71a3-4cb2-b932-94277a421fcc"} {"msg":"[1]R generation[0..12]: [187, 189, 191, 191, 186, 191, 188, 191, 191, 188, 188, 180]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:51.181144465Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1901,"":"downstairs","":"downstairs","session_id":"29b3e9ab-71a3-4cb2-b932-94277a421fcc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:51.181155325Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1901,"":"downstairs","":"downstairs","session_id":"29b3e9ab-71a3-4cb2-b932-94277a421fcc"} {"msg":"[2]R flush_numbers[0..12]: [318, 324, 329, 329, 316, 329, 323, 329, 329, 321, 321, 306]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:51.181165865Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1901,"":"downstairs","":"downstairs","session_id":"29b3e9ab-71a3-4cb2-b932-94277a421fcc"} {"msg":"[2]R generation[0..12]: [187, 189, 191, 191, 186, 191, 188, 191, 191, 188, 188, 180]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:51.181176394Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1901,"":"downstairs","":"downstairs","session_id":"29b3e9ab-71a3-4cb2-b932-94277a421fcc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:51.181187214Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1901,"":"downstairs","":"downstairs","session_id":"29b3e9ab-71a3-4cb2-b932-94277a421fcc"} {"msg":"Max found gen is 192","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:51.181196554Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1901,"":"downstairs","":"downstairs","session_id":"29b3e9ab-71a3-4cb2-b932-94277a421fcc"} {"msg":"Generation requested: 192 >= found:192","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:51.181206264Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1901,"":"downstairs","":"downstairs","session_id":"29b3e9ab-71a3-4cb2-b932-94277a421fcc"} {"msg":"Next flush: 330","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:51.181215533Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1901,"":"downstairs","":"downstairs","session_id":"29b3e9ab-71a3-4cb2-b932-94277a421fcc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:51.181224843Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1901,"":"downstairs","":"downstairs","session_id":"29b3e9ab-71a3-4cb2-b932-94277a421fcc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:51.181234473Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1901,"session_id":"29b3e9ab-71a3-4cb2-b932-94277a421fcc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:51.181245022Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1901,"client":"0","":"downstairs","session_id":"29b3e9ab-71a3-4cb2-b932-94277a421fcc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:51.181254632Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1901,"client":"1","":"downstairs","session_id":"29b3e9ab-71a3-4cb2-b932-94277a421fcc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:51.181264302Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1901,"client":"2","":"downstairs","session_id":"29b3e9ab-71a3-4cb2-b932-94277a421fcc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:51.181274022Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1901,"session_id":"29b3e9ab-71a3-4cb2-b932-94277a421fcc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:51.181283031Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1901,"session_id":"29b3e9ab-71a3-4cb2-b932-94277a421fcc"} {"msg":"3129432b-133b-40eb-aec3-dc733ae2dd03 is now active with session: 29b3e9ab-71a3-4cb2-b932-94277a421fcc","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:51.181292861Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 pid":1901,"session_id":"29b3e9ab-71a3-4cb2-b932-94277a421fcc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:51.181306181Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1901,"session_id":"29b3e9ab-71a3-4cb2-b932-94277a421fcc"} {"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:51.18131526Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1901} {"msg":"Activated sub_volume 3129432b-133b-40eb-aec3-dc733ae2dd03","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:51.18132349Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1901} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:51.18133167Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1901,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:51.181568313Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1901,"task":"crutest"} 01/30 Write block 275 len 16384 data: 10 13 11 13 02/30 Read block 22 len 32768 03/30 Read block 5 len 28672 04/30 Write block 117 len 32768 data: 15 14 14 12 13 16 18 16 05/30 Read block 576 len 16384 06/30 Write block 429 len 20480 data: 19 16 15 15 11 07/30 Flush 08/30 Write block 421 len 12288 data: 12 13 13 09/30 Flush 10/30 Write block 116 len 4096 data: 17 11/30 Read block 260 len 12288 12/30 Read block 9 len 36864 13/30 Read block 66 len 24576 14/30 Write block 318 len 36864 data: 10 13 12 16 15 17 22 23 23 15/30 Flush 16/30 Write block 45 len 40960 data: 11 11 10 11 9 14 13 11 11 12 17/30 Write block 252 len 32768 data: 10 12 11 16 18 15 16 15 18/30 Write block 48 len 12288 data: 12 10 15 19/30 Read block 14 len 16384 20/30 Read block 78 len 20480 21/30 Write block 236 len 36864 data: 11 13 12 13 14 17 17 16 16 22/30 Write block 9 len 40960 data: 17 17 14 12 14 14 13 11 11 12 23/30 Read block 403 len 32768 24/30 Write block 167 len 32768 data: 16 14 15 17 16 19 20 17 25/30 Read block 299 len 36864 26/30 Write block 49 len 40960 data: 11 16 14 12 12 13 15 16 14 16 27/30 Write block 85 len 24576 data: 12 12 13 12 13 13 28/30 Read block 274 len 36864 29/30 Write block 74 len 20480 data: 19 18 17 19 21 30/30 Write block 245 len 24576 data: 16 18 19 18 15 13 ---------------------------------------------------------------- Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Sent Done false 1001 Acked Read 8 Done Sent Done false 1002 Acked Read 7 Done Sent Done false 1003 Acked Write 8 Done Sent Done false 1004 Acked Read 4 Done Sent Done false 1005 Acked Write 5 Done Sent Done false 1006 Acked Flush 0 Done Sent Done false 1007 Acked Write 3 Done Sent Done false 1008 Acked Flush 0 Done Sent Done false 1009 Acked Write 1 Done Sent Done false 1010 Acked Read 3 Done Sent Done false 1011 Acked Read 9 Done Sent Done false 1012 Acked Read 6 Done Sent Done false 1013 Acked Write 9 Done Sent Done false 1014 Acked Flush 0 Done Sent Done false 1015 Acked Write 10 Done Sent Done false 1016 Acked Write 8 Done Sent Done false 1017 Acked Write 3 Done Sent Done false 1018 Acked Read 4 Done Sent Done false 1019 Acked Read 5 Done Sent Done false 1020 Acked Write 9 Done Sent Done false 1021 Acked Write 10 Done Sent Done false 1022 Acked Read 8 Done Sent Done false 1023 Acked Write 8 Done Sent Done false 1024 Acked Read 9 Done Sent Done false 1025 Acked Write 10 Done Sent Done false 1026 Acked Write 6 Done Sent Done false 1027 Acked Read 9 Done Sent Done false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 30 2 34 Done 28 0 28 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) None Some(JobId(1014)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:48:51.190793477Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1901} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:51.190968762Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1901,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 187 187 187 318 318 318 T F T <--- 1 020-039 189 189 189 324 324 324 F F F 2 040-059 191 191 191 329 329 329 T F T <--- 3 060-079 191 191 191 329 329 329 T F T <--- 4 080-099 186 186 186 316 316 316 T F T <--- 5 100-119 192 191 192 332 329 332 F F F <--- 6 120-139 192 188 192 330 323 330 F F F <--- 7 140-159 191 191 191 329 329 329 F F F 8 160-179 191 191 191 329 329 329 T F T <--- 9 180-199 188 188 188 321 321 321 F F F 10 200-219 188 188 188 321 321 321 F F F 11 220-239 180 180 180 306 306 306 T F T <--- 12 240-259 186 186 186 317 317 317 T F T <--- 13 260-279 192 191 192 330 329 330 F F F <--- 14 280-299 186 186 186 317 317 317 F F F 15 300-319 192 191 192 332 329 332 F F F <--- 16 320-339 192 184 192 332 314 332 F F F <--- 17 340-359 191 191 191 329 329 329 F F F 18 360-379 191 191 191 329 329 329 F F F 19 380-399 186 186 186 316 316 316 F F F 20 400-419 190 190 190 327 327 327 F F F 21 420-439 192 191 192 331 329 331 F F F <--- 22 440-459 190 190 190 327 327 327 F F F 23 460-479 178 178 178 300 300 300 F F F 24 480-499 187 187 187 318 318 318 F F F 25 500-519 183 183 183 311 311 311 F F F 26 520-539 191 191 191 329 329 329 F F F 27 540-559 189 189 189 324 324 324 F F F 28 560-579 180 180 180 306 306 306 F F F 29 580-599 184 184 184 313 313 313 F F F Max gen: 192, Max flush: 332 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 96 Verifying data now No change after verify Loop: 96 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 193 193 193 333 333 333 F F F 1 020-039 189 189 189 324 324 324 F F F 2 040-059 193 193 193 333 333 333 F F F 3 060-079 193 193 193 333 333 333 F F F 4 080-099 193 193 193 333 333 333 F F F 5 100-119 192 192 192 332 332 332 F F F 6 120-139 192 192 192 330 330 330 F F F 7 140-159 191 191 191 329 329 329 F F F 8 160-179 193 193 193 333 333 333 F F F 9 180-199 188 188 188 321 321 321 F F F 10 200-219 188 188 188 321 321 321 F F F 11 220-239 193 193 193 333 333 333 F F F 12 240-259 193 193 193 333 333 333 F F F 13 260-279 192 192 192 330 330 330 F F F 14 280-299 186 186 186 317 317 317 F F F 15 300-319 192 192 192 332 332 332 F F F 16 320-339 192 192 192 332 332 332 F F F 17 340-359 191 191 191 329 329 329 F F F 18 360-379 191 191 191 329 329 329 F F F 19 380-399 186 186 186 316 316 316 F F F 20 400-419 190 190 190 327 327 327 F F F 21 420-439 192 192 192 331 331 331 F F F 22 440-459 190 190 190 327 327 327 F F F 23 460-479 178 178 178 300 300 300 F F F 24 480-499 187 187 187 318 318 318 F F F 25 500-519 183 183 183 311 311 311 F F F 26 520-539 191 191 191 329 329 329 F F F 27 540-559 189 189 189 324 324 324 F F F 28 560-579 180 180 180 306 306 306 F F F 29 580-599 184 184 184 313 313 313 F F F Max gen: 193, Max flush: 333 Begin loop 97 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:55.443225657Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1909,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:55.451367312Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1909,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:55.451675563Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1909} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:55.451701493Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1909} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:55.451712332Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1909} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:55.451723862Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1909} {"msg":"Crucible aafbe89d-6e34-4dea-ae0f-06c636fcd54a has session id: b947b2fd-bc51-4e29-95ad-3ffb4fb98940","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:55.452048393Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1909,"session_id":"b947b2fd-bc51-4e29-95ad-3ffb4fb98940"} {"msg":"Upstairs opts: Upstairs UUID: aafbe89d-6e34-4dea-ae0f-06c636fcd54a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:55.452077012Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1909,"session_id":"b947b2fd-bc51-4e29-95ad-3ffb4fb98940"} {"msg":"Crucible stats registered with UUID: aafbe89d-6e34-4dea-ae0f-06c636fcd54a","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:55.452087592Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1909,"session_id":"b947b2fd-bc51-4e29-95ad-3ffb4fb98940"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:48:55.452096761Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1909,"session_id":"b947b2fd-bc51-4e29-95ad-3ffb4fb98940"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:55.452249377Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1909,"":"io task","client":"1","":"downstairs","session_id":"b947b2fd-bc51-4e29-95ad-3ffb4fb98940"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:55.452280646Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1909,"":"io task","client":"0","":"downstairs","session_id":"b947b2fd-bc51-4e29-95ad-3ffb4fb98940"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:55.452291366Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1909,"":"io task","client":"2","":"downstairs","session_id":"b947b2fd-bc51-4e29-95ad-3ffb4fb98940"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:55.452419202Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1909,"task":"crutest"} {"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:55.452434302Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1909} {"msg":"aafbe89d-6e34-4dea-ae0f-06c636fcd54a active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:55.452449991Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1909,"session_id":"b947b2fd-bc51-4e29-95ad-3ffb4fb98940"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:55.452458811Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1909,"client":"0","":"downstairs","session_id":"b947b2fd-bc51-4e29-95ad-3ffb4fb98940"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:55.452468401Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1909,"client":"1","":"downstairs","session_id":"b947b2fd-bc51-4e29-95ad-3ffb4fb98940"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:55.45247792Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1909,"client":"2","":"downstairs","session_id":"b947b2fd-bc51-4e29-95ad-3ffb4fb98940"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:55.45248953Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1909,"":"io task","client":"2","":"downstairs","session_id":"b947b2fd-bc51-4e29-95ad-3ffb4fb98940"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:55.45249949Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1909,"":"io task","client":"0","":"downstairs","session_id":"b947b2fd-bc51-4e29-95ad-3ffb4fb98940"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:55.452509289Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1909,"":"io task","client":"1","":"downstairs","session_id":"b947b2fd-bc51-4e29-95ad-3ffb4fb98940"} {"msg":"ds_connection connected from Ok(127.0.0.1:55725)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:55.452569558Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1909,"":"io task","client":"2","":"downstairs","session_id":"b947b2fd-bc51-4e29-95ad-3ffb4fb98940"} {"msg":"ds_connection connected from Ok(127.0.0.1:56096)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:55.452586947Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1909,"":"io task","client":"0","":"downstairs","session_id":"b947b2fd-bc51-4e29-95ad-3ffb4fb98940"} {"msg":"ds_connection connected from Ok(127.0.0.1:42241)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:55.452597437Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1909,"":"io task","client":"1","":"downstairs","session_id":"b947b2fd-bc51-4e29-95ad-3ffb4fb98940"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:55.453313936Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1909,"client":"0","":"downstairs","session_id":"b947b2fd-bc51-4e29-95ad-3ffb4fb98940"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:55.453338286Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1909,"client":"1","":"downstairs","session_id":"b947b2fd-bc51-4e29-95ad-3ffb4fb98940"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:55.453349435Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1909,"client":"2","":"downstairs","session_id":"b947b2fd-bc51-4e29-95ad-3ffb4fb98940"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:55.453609858Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1909,"session_id":"b947b2fd-bc51-4e29-95ad-3ffb4fb98940"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:55.453633717Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1909,"session_id":"b947b2fd-bc51-4e29-95ad-3ffb4fb98940"} {"msg":"[0]R flush_numbers[0..12]: [333, 324, 333, 333, 333, 332, 330, 329, 333, 321, 321, 333]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:55.453720425Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1909,"":"downstairs","":"downstairs","session_id":"b947b2fd-bc51-4e29-95ad-3ffb4fb98940"} {"msg":"[0]R generation[0..12]: [193, 189, 193, 193, 193, 192, 192, 191, 193, 188, 188, 193]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:55.453732224Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1909,"":"downstairs","":"downstairs","session_id":"b947b2fd-bc51-4e29-95ad-3ffb4fb98940"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:55.453743174Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1909,"":"downstairs","":"downstairs","session_id":"b947b2fd-bc51-4e29-95ad-3ffb4fb98940"} {"msg":"[1]R flush_numbers[0..12]: [333, 324, 333, 333, 333, 332, 330, 329, 333, 321, 321, 333]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:55.453753644Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1909,"":"downstairs","":"downstairs","session_id":"b947b2fd-bc51-4e29-95ad-3ffb4fb98940"} {"msg":"[1]R generation[0..12]: [193, 189, 193, 193, 193, 192, 192, 191, 193, 188, 188, 193]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:55.453766603Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1909,"":"downstairs","":"downstairs","session_id":"b947b2fd-bc51-4e29-95ad-3ffb4fb98940"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:55.453777443Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1909,"":"downstairs","":"downstairs","session_id":"b947b2fd-bc51-4e29-95ad-3ffb4fb98940"} {"msg":"[2]R flush_numbers[0..12]: [333, 324, 333, 333, 333, 332, 330, 329, 333, 321, 321, 333]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:55.453787993Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1909,"":"downstairs","":"downstairs","session_id":"b947b2fd-bc51-4e29-95ad-3ffb4fb98940"} {"msg":"[2]R generation[0..12]: [193, 189, 193, 193, 193, 192, 192, 191, 193, 188, 188, 193]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:55.453798452Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1909,"":"downstairs","":"downstairs","session_id":"b947b2fd-bc51-4e29-95ad-3ffb4fb98940"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:55.453809272Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1909,"":"downstairs","":"downstairs","session_id":"b947b2fd-bc51-4e29-95ad-3ffb4fb98940"} {"msg":"Max found gen is 194","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:55.453818592Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1909,"":"downstairs","":"downstairs","session_id":"b947b2fd-bc51-4e29-95ad-3ffb4fb98940"} {"msg":"Generation requested: 194 >= found:194","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:55.453828421Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1909,"":"downstairs","":"downstairs","session_id":"b947b2fd-bc51-4e29-95ad-3ffb4fb98940"} {"msg":"Next flush: 334","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:55.453837741Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1909,"":"downstairs","":"downstairs","session_id":"b947b2fd-bc51-4e29-95ad-3ffb4fb98940"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:55.453847061Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1909,"":"downstairs","":"downstairs","session_id":"b947b2fd-bc51-4e29-95ad-3ffb4fb98940"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:55.453856881Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1909,"session_id":"b947b2fd-bc51-4e29-95ad-3ffb4fb98940"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:55.45386578Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1909,"client":"0","":"downstairs","session_id":"b947b2fd-bc51-4e29-95ad-3ffb4fb98940"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:55.45387552Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1909,"client":"1","":"downstairs","session_id":"b947b2fd-bc51-4e29-95ad-3ffb4fb98940"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:55.45388508Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1909,"client":"2","":"downstairs","session_id":"b947b2fd-bc51-4e29-95ad-3ffb4fb98940"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:55.453894809Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1909,"session_id":"b947b2fd-bc51-4e29-95ad-3ffb4fb98940"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:55.453903829Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1909,"session_id":"b947b2fd-bc51-4e29-95ad-3ffb4fb98940"} {"msg":"aafbe89d-6e34-4dea-ae0f-06c636fcd54a is now active with session: b947b2fd-bc51-4e29-95ad-3ffb4fb98940","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:55.453915209Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1909,"session_id":"b947b2fd-bc51-4e29-95ad-3ffb4fb98940"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:55.453923709Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1909,"session_id":"b947b2fd-bc51-4e29-95ad-3ffb4fb98940"} {"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:55.453932728Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1909} {"msg":"Activated sub_volume aafbe89d-6e34-4dea-ae0f-06c636fcd54a","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:55.453940628Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1909} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:55.453948708Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1909,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:55.454307298Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1909,"task":"crutest"} 01/30 Write block 467 len 40960 data: 7 10 10 11 10 8 8 12 16 21 02/30 Read block 155 len 24576 03/30 Write block 409 len 4096 data: 12 04/30 Write block 104 len 24576 data: 8 8 8 9 8 8 05/30 Read block 393 len 28672 06/30 Write block 572 len 16384 data: 12 13 13 14 07/30 Flush 08/30 Write block 63 len 28672 data: 14 12 10 10 11 9 10 09/30 Read block 375 len 24576 10/30 Write block 442 len 32768 data: 9 9 11 10 10 8 9 11 11/30 Write block 54 len 28672 data: 14 16 17 15 17 14 15 12/30 Write block 173 len 4096 data: 21 13/30 Read block 39 len 8192 14/30 Read block 52 len 40960 15/30 Read block 145 len 16384 16/30 Write block 289 len 8192 data: 15 13 17/30 Write block 11 len 36864 data: 15 13 15 15 14 12 12 13 9 18/30 Read block 480 len 28672 19/30 Read block 236 len 16384 20/30 Read block 281 len 4096 21/30 Read block 75 len 8192 22/30 Read block 590 len 16384 23/30 Write block 564 len 12288 data: 13 12 11 24/30 Write block 337 len 32768 data: 15 9 8 8 10 10 11 12 25/30 Write block 397 len 20480 data: 14 12 18 20 19 26/30 Write block 389 len 24576 data: 12 15 14 14 15 15 27/30 Write block 427 len 28672 data: 15 14 20 17 16 16 12 28/30 Write block 498 len 16384 data: 10 11 15 20 29/30 Write block 462 len 24576 data: 10 10 9 7 7 8 30/30 Read block 359 len 28672 ---------------------------------------------------------------- Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Read 6 Done Done Done false 1002 Acked Write 1 Done Done Done false 1003 Acked Write 6 Done Done Done false 1004 Acked Read 7 Done Done Sent false 1005 Acked Write 4 Done Done Sent false 1006 Acked Flush 0 Done Done Sent false 1007 Acked Write 7 Done Done Sent false 1008 Acked Read 6 Done Done Sent false 1009 Acked Write 8 Done Done Sent false 1010 Acked Write 7 Done Done Sent false 1011 Acked Write 1 Done Done Sent false 1012 Acked Read 2 Done Done Sent false 1013 Acked Read 10 Done Done Sent false 1014 Acked Read 4 Done Done Sent false 1015 Acked Write 2 Done Done Sent false 1016 Acked Write 9 Done Done Sent false 1017 Acked Read 7 Done Done Sent false 1018 Acked Read 4 Done Done Sent false 1019 Acked Read 1 Done Done Sent false 1020 Acked Read 2 Done Done Sent false 1021 Acked Read 4 Done Done Sent false 1022 Acked Write 3 Done Done Sent false 1023 Acked Write 8 Done Done Sent false 1024 Acked Write 5 Done Done Sent false 1025 Acked Write 6 Done Done Sent false 1026 Acked Write 7 Done Done Sent false 1027 Acked Write 4 Done Done Sent false 1028 Acked Write 6 Done Done Sent false 1029 Acked Read 7 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 26 26 Done 30 30 4 64 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1006)) Some(JobId(1006)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:48:55.461983736Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1909} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:55.462177971Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1909,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 193 193 193 333 333 333 T T F <--- 1 020-039 189 189 189 324 324 324 F F F 2 040-059 193 193 193 333 333 333 T T F <--- 3 060-079 193 193 193 333 333 333 T T F <--- 4 080-099 193 193 193 333 333 333 F F F 5 100-119 194 194 192 334 334 332 F F T <--- 6 120-139 192 192 192 330 330 330 F F F 7 140-159 191 191 191 329 329 329 F F F 8 160-179 193 193 193 333 333 333 T T F <--- 9 180-199 188 188 188 321 321 321 F F F 10 200-219 188 188 188 321 321 321 F F F 11 220-239 193 193 193 333 333 333 F F F 12 240-259 193 193 193 333 333 333 F F F 13 260-279 192 192 192 330 330 330 F F F 14 280-299 186 186 186 317 317 317 T T F <--- 15 300-319 192 192 192 332 332 332 F F F 16 320-339 192 192 192 332 332 332 T T F <--- 17 340-359 191 191 191 329 329 329 T T F <--- 18 360-379 191 191 191 329 329 329 F F F 19 380-399 186 186 186 316 316 316 T T F <--- 20 400-419 194 194 190 334 334 327 T T T <--- 21 420-439 192 192 192 331 331 331 T T F <--- 22 440-459 190 190 190 327 327 327 T T F <--- 23 460-479 194 194 178 334 334 300 T T T <--- 24 480-499 187 187 187 318 318 318 T T F <--- 25 500-519 183 183 183 311 311 311 T T F <--- 26 520-539 191 191 191 329 329 329 F F F 27 540-559 189 189 189 324 324 324 F F F 28 560-579 194 194 180 334 334 306 T T F <--- 29 580-599 184 184 184 313 313 313 F F F Max gen: 194, Max flush: 334 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 97 Verifying data now No change after verify Loop: 97 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 195 195 195 335 335 335 F F F 1 020-039 189 189 189 324 324 324 F F F 2 040-059 195 195 195 335 335 335 F F F 3 060-079 195 195 195 335 335 335 F F F 4 080-099 193 193 193 333 333 333 F F F 5 100-119 194 194 194 334 334 334 F F F 6 120-139 192 192 192 330 330 330 F F F 7 140-159 191 191 191 329 329 329 F F F 8 160-179 195 195 195 335 335 335 F F F 9 180-199 188 188 188 321 321 321 F F F 10 200-219 188 188 188 321 321 321 F F F 11 220-239 193 193 193 333 333 333 F F F 12 240-259 193 193 193 333 333 333 F F F 13 260-279 192 192 192 330 330 330 F F F 14 280-299 195 195 195 335 335 335 F F F 15 300-319 192 192 192 332 332 332 F F F 16 320-339 195 195 195 335 335 335 F F F 17 340-359 195 195 195 335 335 335 F F F 18 360-379 191 191 191 329 329 329 F F F 19 380-399 195 195 195 335 335 335 F F F 20 400-419 195 195 195 335 335 335 F F F 21 420-439 195 195 195 335 335 335 F F F 22 440-459 195 195 195 335 335 335 F F F 23 460-479 195 195 195 335 335 335 F F F 24 480-499 195 195 195 335 335 335 F F F 25 500-519 195 195 195 335 335 335 F F F 26 520-539 191 191 191 329 329 329 F F F 27 540-559 189 189 189 324 324 324 F F F 28 560-579 195 195 195 335 335 335 F F F 29 580-599 184 184 184 313 313 313 F F F Max gen: 195, Max flush: 335 Begin loop 98 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:59.72467233Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1917,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:48:59.731717757Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1917,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:59.731773526Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1917,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:59.73856063Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1917,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:59.738821112Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1917} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:59.738846822Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1917} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:59.738855291Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1917} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:59.738866671Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1917} {"msg":"Crucible 608c1010-f8d3-4737-81f3-dc4f4405515c has session id: 0ba68058-5db3-4c47-87a7-7355c0c602d0","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:59.739183722Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1917,"session_id":"0ba68058-5db3-4c47-87a7-7355c0c602d0"} {"msg":"Upstairs opts: Upstairs UUID: 608c1010-f8d3-4737-81f3-dc4f4405515c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:59.739214021Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1917,"session_id":"0ba68058-5db3-4c47-87a7-7355c0c602d0"} {"msg":"Crucible stats registered with UUID: 608c1010-f8d3-4737-81f3-dc4f4405515c","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:59.739224661Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1917,"session_id":"0ba68058-5db3-4c47-87a7-7355c0c602d0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:48:59.73923362Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1917,"session_id":"0ba68058-5db3-4c47-87a7-7355c0c602d0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:59.739364817Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1917,"":"io task","client":"0","":"downstairs","session_id":"0ba68058-5db3-4c47-87a7-7355c0c602d0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:59.739396716Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1917,"":"io task","client":"1","":"downstairs","session_id":"0ba68058-5db3-4c47-87a7-7355c0c602d0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:59.739407875Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1917,"":"io task","client":"2","":"downstairs","session_id":"0ba68058-5db3-4c47-87a7-7355c0c602d0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:59.739525862Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1917,"task":"crutest"} {"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:59.739541742Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1917} {"msg":"608c1010-f8d3-4737-81f3-dc4f4405515c active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:59.739565251Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1917,"session_id":"0ba68058-5db3-4c47-87a7-7355c0c602d0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:59.739574581Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1917,"client":"0","":"downstairs","session_id":"0ba68058-5db3-4c47-87a7-7355c0c602d0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:59.73958434Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1917,"client":"1","":"downstairs","session_id":"0ba68058-5db3-4c47-87a7-7355c0c602d0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:59.73959422Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1917,"client":"2","":"downstairs","session_id":"0ba68058-5db3-4c47-87a7-7355c0c602d0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:59.73960375Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1917,"":"io task","client":"2","":"downstairs","session_id":"0ba68058-5db3-4c47-87a7-7355c0c602d0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:59.739632809Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1917,"":"io task","client":"0","":"downstairs","session_id":"0ba68058-5db3-4c47-87a7-7355c0c602d0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:59.739648428Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1917,"":"io task","client":"1","":"downstairs","session_id":"0ba68058-5db3-4c47-87a7-7355c0c602d0"} {"msg":"ds_connection connected from Ok(127.0.0.1:49434)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:59.739698837Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1917,"":"io task","client":"2","":"downstairs","session_id":"0ba68058-5db3-4c47-87a7-7355c0c602d0"} {"msg":"ds_connection connected from Ok(127.0.0.1:35383)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:59.739729076Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1917,"":"io task","client":"1","":"downstairs","session_id":"0ba68058-5db3-4c47-87a7-7355c0c602d0"} {"msg":"ds_connection connected from Ok(127.0.0.1:56417)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:59.739742736Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1917,"":"io task","client":"0","":"downstairs","session_id":"0ba68058-5db3-4c47-87a7-7355c0c602d0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:59.740462095Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1917,"client":"0","":"downstairs","session_id":"0ba68058-5db3-4c47-87a7-7355c0c602d0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:59.740488404Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1917,"client":"1","":"downstairs","session_id":"0ba68058-5db3-4c47-87a7-7355c0c602d0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:59.740531203Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1917,"client":"2","":"downstairs","session_id":"0ba68058-5db3-4c47-87a7-7355c0c602d0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:59.740801275Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1917,"session_id":"0ba68058-5db3-4c47-87a7-7355c0c602d0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:59.740838144Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1917,"session_id":"0ba68058-5db3-4c47-87a7-7355c0c602d0"} {"msg":"[0]R flush_numbers[0..12]: [335, 324, 335, 335, 333, 334, 330, 329, 335, 321, 321, 333]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:59.740904682Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1917,"":"downstairs","":"downstairs","session_id":"0ba68058-5db3-4c47-87a7-7355c0c602d0"} {"msg":"[0]R generation[0..12]: [195, 189, 195, 195, 193, 194, 192, 191, 195, 188, 188, 193]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:59.740915842Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1917,"":"downstairs","":"downstairs","session_id":"0ba68058-5db3-4c47-87a7-7355c0c602d0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:59.740926792Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1917,"":"downstairs","":"downstairs","session_id":"0ba68058-5db3-4c47-87a7-7355c0c602d0"} {"msg":"[1]R flush_numbers[0..12]: [335, 324, 335, 335, 333, 334, 330, 329, 335, 321, 321, 333]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:59.740937361Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1917,"":"downstairs","":"downstairs","session_id":"0ba68058-5db3-4c47-87a7-7355c0c602d0"} {"msg":"[1]R generation[0..12]: [195, 189, 195, 195, 193, 194, 192, 191, 195, 188, 188, 193]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:59.740947811Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1917,"":"downstairs","":"downstairs","session_id":"0ba68058-5db3-4c47-87a7-7355c0c602d0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:59.740958791Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1917,"":"downstairs","":"downstairs","session_id":"0ba68058-5db3-4c47-87a7-7355c0c602d0"} {"msg":"[2]R flush_numbers[0..12]: [335, 324, 335, 335, 333, 334, 330, 329, 335, 321, 321, 333]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:59.74097186Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1917,"":"downstairs","":"downstairs","session_id":"0ba68058-5db3-4c47-87a7-7355c0c602d0"} {"msg":"[2]R generation[0..12]: [195, 189, 195, 195, 193, 194, 192, 191, 195, 188, 188, 193]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:59.74098243Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1917,"":"downstairs","":"downstairs","session_id":"0ba68058-5db3-4c47-87a7-7355c0c602d0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:59.7409931Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1917,"":"downstairs","":"downstairs","session_id":"0ba68058-5db3-4c47-87a7-7355c0c602d0"} {"msg":"Max found gen is 196","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:59.741002719Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1917,"":"downstairs","":"downstairs","session_id":"0ba68058-5db3-4c47-87a7-7355c0c602d0"} {"msg":"Generation requested: 196 >= found:196","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:59.741012319Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1917,"":"downstairs","":"downstairs","session_id":"0ba68058-5db3-4c47-87a7-7355c0c602d0"} {"msg":"Next flush: 336","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:59.741021789Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1917,"":"downstairs","":"downstairs","session_id":"0ba68058-5db3-4c47-87a7-7355c0c602d0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:59.741031169Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1917,"":"downstairs","":"downstairs","session_id":"0ba68058-5db3-4c47-87a7-7355c0c602d0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:59.741040908Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1917,"session_id":"0ba68058-5db3-4c47-87a7-7355c0c602d0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:59.741049568Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1917,"client":"0","":"downstairs","session_id":"0ba68058-5db3-4c47-87a7-7355c0c602d0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:59.741059148Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1917,"client":"1","":"downstairs","session_id":"0ba68058-5db3-4c47-87a7-7355c0c602d0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:59.741068748Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1917,"client":"2","":"downstairs","session_id":"0ba68058-5db3-4c47-87a7-7355c0c602d0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:59.741078697Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1917,"session_id":"0ba68058-5db3-4c47-87a7-7355c0c602d0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:59.741087757Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1917,"session_id":"0ba68058-5db3-4c47-87a7-7355c0c602d0"} {"msg":"608c1010-f8d3-4737-81f3-dc4f4405515c is now active with session: 0ba68058-5db3-4c47-87a7-7355c0c602d0","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:59.741097687Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1917,"session_id":"0ba68058-5db3-4c47-87a7-7355c0c602d0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:59.741106446Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1917,"session_id":"0ba68058-5db3-4c47-87a7-7355c0c602d0"} {"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:59.741115596Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1917} {"msg":"Activated sub_volume 608c1010-f8d3-4737-81f3-dc4f4405515c","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:59.741124976Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1917} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:59.741132956Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1917,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:59.741478536Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1917,"task":"crutest"} 01/30 Write block 439 len 36864 data: 9 11 10 10 10 12 11 11 9 02/30 Read block 124 len 12288 03/30 Write block 200 len 16384 data: 10 17 20 19 04/30 Read block 580 len 12288 05/30 Write block 9 len 16384 data: 18 18 16 14 06/30 Write block 438 len 32768 data: 10 10 12 11 11 11 13 12 07/30 Write block 344 len 12288 data: 13 13 14 08/30 Read block 238 len 28672 09/30 Write block 20 len 24576 data: 9 8 14 12 13 13 10/30 Read block 109 len 32768 11/30 Read block 502 len 36864 12/30 Read block 343 len 24576 13/30 Read block 411 len 28672 14/30 Read block 173 len 20480 15/30 Write block 30 len 4096 data: 10 16/30 Read block 200 len 24576 17/30 Flush 18/30 Write block 463 len 24576 data: 11 10 8 8 9 11 19/30 Read block 371 len 8192 20/30 Write block 298 len 32768 data: 13 11 12 13 10 11 11 13 21/30 Flush 22/30 Flush 23/30 Write block 486 len 28672 data: 16 18 21 21 18 18 15 24/30 Read block 121 len 24576 25/30 Read block 364 len 4096 26/30 Write block 50 len 4096 data: 17 27/30 Write block 546 len 8192 data: 13 13 28/30 Write block 172 len 36864 data: 20 22 18 16 16 16 15 18 21 29/30 Read block 263 len 4096 30/30 Write block 210 len 24576 data: 16 16 17 20 18 14 ---------------------------------------------------------------- Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Read 3 Done Done Done false 1002 Acked Write 4 Done Done Done false 1003 Acked Read 3 Sent Done Done false 1004 Acked Write 4 Sent Done Done false 1005 Acked Write 8 Sent Done Done false 1006 Acked Write 3 Sent Done Done false 1007 Acked Read 7 Sent Done Done false 1008 Acked Write 6 Sent Done Done false 1009 Acked Read 8 Sent Done Done false 1010 Acked Read 9 Sent Done Done false 1011 Acked Read 6 Sent Done Done false 1012 Acked Read 7 Sent Done Done false 1013 Acked Read 5 Sent Done Done false 1014 Acked Write 1 Sent Done Done false 1015 Acked Read 6 Sent Done Done false 1016 Acked Flush 0 Sent Done Done false 1017 Acked Write 6 Sent Done Done false 1018 Acked Read 2 Sent Done Done false 1019 Acked Write 8 Sent Done Done false 1020 Acked Flush 0 Sent Done Done false 1021 Acked Flush 0 Sent Done Done false 1022 Acked Write 7 Sent Done Done false 1023 Acked Read 6 Sent Done Done false 1024 Acked Read 1 Sent Done Done false 1025 Acked Write 1 Sent Done Done false 1026 Acked Write 2 Sent Done Done false 1027 Acked Write 9 Sent Done Done false 1028 Acked Read 1 Sent Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 27 1 1 29 Done 3 29 29 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:48:59.750588473Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1917} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:48:59.750782517Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1917,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 195 196 196 335 336 336 F F F <--- 1 020-039 189 196 196 324 336 336 F F F <--- 2 040-059 195 195 195 335 335 335 F T T <--- 3 060-079 195 195 195 335 335 335 F F F 4 080-099 193 193 193 333 333 333 F F F 5 100-119 194 194 194 334 334 334 F F F 6 120-139 192 192 192 330 330 330 F F F 7 140-159 191 191 191 329 329 329 F F F 8 160-179 195 195 195 335 335 335 F T T <--- 9 180-199 188 188 188 321 321 321 F T T <--- 10 200-219 188 196 196 321 336 336 T T T <--- 11 220-239 193 193 193 333 333 333 F F F 12 240-259 193 193 193 333 333 333 F F F 13 260-279 192 192 192 330 330 330 F F F 14 280-299 195 196 196 335 337 337 F F F <--- 15 300-319 192 196 196 332 337 337 F F F <--- 16 320-339 195 195 195 335 335 335 F F F 17 340-359 195 196 196 335 336 336 F F F <--- 18 360-379 191 191 191 329 329 329 F F F 19 380-399 195 195 195 335 335 335 F F F 20 400-419 195 195 195 335 335 335 F F F 21 420-439 195 196 196 335 336 336 T F F <--- 22 440-459 195 196 196 335 336 336 T F F <--- 23 460-479 195 196 196 335 337 337 F F F <--- 24 480-499 195 195 195 335 335 335 F T T <--- 25 500-519 195 195 195 335 335 335 F F F 26 520-539 191 191 191 329 329 329 F F F 27 540-559 189 189 189 324 324 324 F T T <--- 28 560-579 195 195 195 335 335 335 F F F 29 580-599 184 184 184 313 313 313 F F F Max gen: 196, Max flush: 337 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 98 Verifying data now No change after verify Loop: 98 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 196 196 196 336 336 336 F F F 1 020-039 196 196 196 336 336 336 F F F 2 040-059 197 197 197 338 338 338 F F F 3 060-079 195 195 195 335 335 335 F F F 4 080-099 193 193 193 333 333 333 F F F 5 100-119 194 194 194 334 334 334 F F F 6 120-139 192 192 192 330 330 330 F F F 7 140-159 191 191 191 329 329 329 F F F 8 160-179 197 197 197 338 338 338 F F F 9 180-199 197 197 197 338 338 338 F F F 10 200-219 197 197 197 338 338 338 F F F 11 220-239 193 193 193 333 333 333 F F F 12 240-259 193 193 193 333 333 333 F F F 13 260-279 192 192 192 330 330 330 F F F 14 280-299 196 196 196 337 337 337 F F F 15 300-319 196 196 196 337 337 337 F F F 16 320-339 195 195 195 335 335 335 F F F 17 340-359 196 196 196 336 336 336 F F F 18 360-379 191 191 191 329 329 329 F F F 19 380-399 195 195 195 335 335 335 F F F 20 400-419 195 195 195 335 335 335 F F F 21 420-439 196 196 196 336 336 336 F F F 22 440-459 196 196 196 336 336 336 F F F 23 460-479 196 196 196 337 337 337 F F F 24 480-499 197 197 197 338 338 338 F F F 25 500-519 195 195 195 335 335 335 F F F 26 520-539 191 191 191 329 329 329 F F F 27 540-559 197 197 197 338 338 338 F F F 28 560-579 195 195 195 335 335 335 F F F 29 580-599 184 184 184 313 313 313 F F F Max gen: 197, Max flush: 338 Begin loop 99 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:49:04.009892344Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1925,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:49:04.018073368Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1925,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:49:04.018382909Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1925} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd6927c220d4f180c3c3e86b37fe26793c41da13\",\n git_commit_timestamp: \"2026-08-03T21:42:13.000000000Z\",\n git_branch: \"alan/raw-to-csv\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:49:04.018406679Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1925} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:49:04.018415449Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1925} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:49:04.018426848Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1925} {"msg":"Crucible 7bca555b-4b4b-4267-959b-0eaf0021ba2a has session id: 01bcb23a-f530-4b6e-a1a3-12459873e3ac","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:49:04.018774038Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1925,"session_id":"01bcb23a-f530-4b6e-a1a3-12459873e3ac"} {"msg":"Upstairs opts: Upstairs UUID: 7bca555b-4b4b-4267-959b-0eaf0021ba2a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:49:04.018802037Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1925,"session_id":"01bcb23a-f530-4b6e-a1a3-12459873e3ac"} {"msg":"Crucible stats registered with UUID: 7bca555b-4b4b-4267-959b-0eaf0021ba2a","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:49:04.018813267Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1925,"session_id":"01bcb23a-f530-4b6e-a1a3-12459873e3ac"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:49:04.018822367Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1925,"session_id":"01bcb23a-f530-4b6e-a1a3-12459873e3ac"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:49:04.018985182Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1925,"":"io task","client":"0","":"downstairs","session_id":"01bcb23a-f530-4b6e-a1a3-12459873e3ac"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:49:04.019018321Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1925,"":"io task","client":"2","":"downstairs","session_id":"01bcb23a-f530-4b6e-a1a3-12459873e3ac"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:49:04.019029371Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1925,"":"io task","client":"1","":"downstairs","session_id":"01bcb23a-f530-4b6e-a1a3-12459873e3ac"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:49:04.019161167Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1925,"task":"crutest"} {"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:49:04.019184126Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1925} {"msg":"7bca555b-4b4b-4267-959b-0eaf0021ba2a active request set","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:49:04.019193046Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1925,"session_id":"01bcb23a-f530-4b6e-a1a3-12459873e3ac"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:49:04.019202066Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1925,"client":"0","":"downstairs","session_id":"01bcb23a-f530-4b6e-a1a3-12459873e3ac"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:49:04.019212286Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1925,"client":"1","":"downstairs","session_id":"01bcb23a-f530-4b6e-a1a3-12459873e3ac"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:49:04.019222015Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1925,"client":"2","":"downstairs","session_id":"01bcb23a-f530-4b6e-a1a3-12459873e3ac"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:49:04.019231675Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1925,"":"io task","client":"2","":"downstairs","session_id":"01bcb23a-f530-4b6e-a1a3-12459873e3ac"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:49:04.019264654Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1925,"":"io task","client":"0","":"downstairs","session_id":"01bcb23a-f530-4b6e-a1a3-12459873e3ac"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:49:04.019286143Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1925,"":"io task","client":"1","":"downstairs","session_id":"01bcb23a-f530-4b6e-a1a3-12459873e3ac"} {"msg":"ds_connection connected from Ok(127.0.0.1:35639)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:49:04.019297063Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1925,"":"io task","client":"2","":"downstairs","session_id":"01bcb23a-f530-4b6e-a1a3-12459873e3ac"} {"msg":"ds_connection connected from Ok(127.0.0.1:38800)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:49:04.019307533Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1925,"":"io task","client":"0","":"downstairs","session_id":"01bcb23a-f530-4b6e-a1a3-12459873e3ac"} {"msg":"ds_connection connected from Ok(127.0.0.1:58854)","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:49:04.01938847Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1925,"":"io task","client":"1","":"downstairs","session_id":"01bcb23a-f530-4b6e-a1a3-12459873e3ac"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:49:04.020013772Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1925,"client":"0","":"downstairs","session_id":"01bcb23a-f530-4b6e-a1a3-12459873e3ac"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:49:04.020032682Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1925,"client":"1","":"downstairs","session_id":"01bcb23a-f530-4b6e-a1a3-12459873e3ac"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:49:04.020172128Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1925,"client":"2","":"downstairs","session_id":"01bcb23a-f530-4b6e-a1a3-12459873e3ac"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:49:04.020400161Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1925,"session_id":"01bcb23a-f530-4b6e-a1a3-12459873e3ac"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:49:04.020409591Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1925,"session_id":"01bcb23a-f530-4b6e-a1a3-12459873e3ac"} {"msg":"[0]R flush_numbers[0..12]: [336, 336, 338, 335, 333, 334, 330, 329, 338, 338, 338, 333]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:49:04.020515118Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1925,"":"downstairs","":"downstairs","session_id":"01bcb23a-f530-4b6e-a1a3-12459873e3ac"} {"msg":"[0]R generation[0..12]: [196, 196, 197, 195, 193, 194, 192, 191, 197, 197, 197, 193]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:49:04.020526238Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1925,"":"downstairs","":"downstairs","session_id":"01bcb23a-f530-4b6e-a1a3-12459873e3ac"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:49:04.020537147Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1925,"":"downstairs","":"downstairs","session_id":"01bcb23a-f530-4b6e-a1a3-12459873e3ac"} {"msg":"[1]R flush_numbers[0..12]: [336, 336, 338, 335, 333, 334, 330, 329, 338, 338, 338, 333]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:49:04.020547647Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1925,"":"downstairs","":"downstairs","session_id":"01bcb23a-f530-4b6e-a1a3-12459873e3ac"} {"msg":"[1]R generation[0..12]: [196, 196, 197, 195, 193, 194, 192, 191, 197, 197, 197, 193]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:49:04.020558157Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1925,"":"downstairs","":"downstairs","session_id":"01bcb23a-f530-4b6e-a1a3-12459873e3ac"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:49:04.020568946Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1925,"":"downstairs","":"downstairs","session_id":"01bcb23a-f530-4b6e-a1a3-12459873e3ac"} {"msg":"[2]R flush_numbers[0..12]: [336, 336, 338, 335, 333, 334, 330, 329, 338, 338, 338, 333]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:49:04.020579546Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1925,"":"downstairs","":"downstairs","session_id":"01bcb23a-f530-4b6e-a1a3-12459873e3ac"} {"msg":"[2]R generation[0..12]: [196, 196, 197, 195, 193, 194, 192, 191, 197, 197, 197, 193]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:49:04.020590056Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1925,"":"downstairs","":"downstairs","session_id":"01bcb23a-f530-4b6e-a1a3-12459873e3ac"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:49:04.020600705Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1925,"":"downstairs","":"downstairs","session_id":"01bcb23a-f530-4b6e-a1a3-12459873e3ac"} {"msg":"Max found gen is 198","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:49:04.020609975Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1925,"":"downstairs","":"downstairs","session_id":"01bcb23a-f530-4b6e-a1a3-12459873e3ac"} {"msg":"Generation requested: 198 >= found:198","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:49:04.020619595Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1925,"":"downstairs","":"downstairs","session_id":"01bcb23a-f530-4b6e-a1a3-12459873e3ac"} {"msg":"Next flush: 339","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:49:04.020630415Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1925,"":"downstairs","":"downstairs","session_id":"01bcb23a-f530-4b6e-a1a3-12459873e3ac"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:49:04.020639664Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1925,"":"downstairs","":"downstairs","session_id":"01bcb23a-f530-4b6e-a1a3-12459873e3ac"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:49:04.020649584Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1925,"session_id":"01bcb23a-f530-4b6e-a1a3-12459873e3ac"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:49:04.020658324Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1925,"client":"0","":"downstairs","session_id":"01bcb23a-f530-4b6e-a1a3-12459873e3ac"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:49:04.020668204Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1925,"client":"1","":"downstairs","session_id":"01bcb23a-f530-4b6e-a1a3-12459873e3ac"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:49:04.020677933Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1925,"client":"2","":"downstairs","session_id":"01bcb23a-f530-4b6e-a1a3-12459873e3ac"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:49:04.020687883Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1925,"session_id":"01bcb23a-f530-4b6e-a1a3-12459873e3ac"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:49:04.020697073Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1925,"session_id":"01bcb23a-f530-4b6e-a1a3-12459873e3ac"} {"msg":"7bca555b-4b4b-4267-959b-0eaf0021ba2a is now active with session: 01bcb23a-f530-4b6e-a1a3-12459873e3ac","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:49:04.020706922Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1925,"session_id":"01bcb23a-f530-4b6e-a1a3-12459873e3ac"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:49:04.020715522Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1925,"session_id":"01bcb23a-f530-4b6e-a1a3-12459873e3ac"} {"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:49:04.020724472Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1925} {"msg":"Activated sub_volume 7bca555b-4b4b-4267-959b-0eaf0021ba2a","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:49:04.020732482Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1925} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:49:04.020740471Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1925,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:49:04.021071172Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1925,"task":"crutest"} 01/30 Write block 567 len 24576 data: 7 9 11 11 13 13 02/30 Read block 399 len 4096 03/30 Flush 04/30 Write block 207 len 16384 data: 13 15 16 17 05/30 Read block 100 len 28672 06/30 Write block 456 len 16384 data: 12 14 13 10 07/30 Read block 273 len 8192 08/30 Write block 574 len 4096 data: 14 09/30 Read block 511 len 16384 10/30 Read block 383 len 36864 11/30 Read block 219 len 20480 12/30 Read block 328 len 24576 13/30 Write block 219 len 20480 data: 13 11 11 13 17 14/30 Read block 393 len 12288 15/30 Write block 118 len 4096 data: 15 16/30 Read block 578 len 20480 17/30 Read block 312 len 20480 18/30 Read block 519 len 40960 19/30 Write block 592 len 24576 data: 10 11 9 9 8 7 20/30 Write block 192 len 12288 data: 17 16 18 21/30 Flush 22/30 Flush 23/30 Write block 487 len 4096 data: 19 24/30 Write block 106 len 4096 data: 9 25/30 Write block 392 len 8192 data: 15 16 26/30 Flush 27/30 Write block 1 len 32768 data: 8 11 8 13 12 15 17 17 28/30 Read block 121 len 20480 29/30 Read block 75 len 16384 30/30 Write block 564 len 24576 data: 14 13 12 8 10 12 ---------------------------------------------------------------- Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Read 1 Done Done Done false 1002 Acked Flush 0 Done Done Sent false 1003 Acked Write 4 Done Done Sent false 1004 Acked Read 7 Done Done Sent false 1005 Acked Write 4 Done Done Sent false 1006 Acked Read 2 Done Done Sent false 1007 Acked Write 1 Done Done Sent false 1008 Acked Read 4 Done Done Sent false 1009 Acked Read 9 Done Done Sent false 1010 Acked Read 5 Done Done Sent false 1011 Acked Read 6 Done Done Sent false 1012 Acked Write 5 Done Done Sent false 1013 Acked Read 3 Done Done Sent false 1014 Acked Write 1 Done Done Sent false 1015 Acked Read 5 Done Done Sent false 1016 Acked Read 5 Done Done Sent false 1017 Acked Read 10 Done Done Sent false 1018 Acked Write 6 Done Done Sent false 1019 Acked Write 3 Done Done Sent false 1020 Acked Flush 0 Done Done Sent false 1021 Acked Flush 0 Done Done Sent false 1022 Acked Write 1 Done Done Sent false 1023 Acked Write 1 Done Done Sent false 1024 Acked Write 2 Done Done Sent false 1025 Acked Flush 0 Done Done Sent false 1026 Acked Write 8 Done Done Sent false 1027 Acked Read 5 Done Done Sent false 1028 Acked Read 4 Done Done Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 28 30 Done 29 29 2 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-08-03T22:49:04.029136839Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1925} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-08-03T22:49:04.029318854Z","hostname":"w-01KZ4WJDV6HRMH059N41G8HZR9","pid":1925,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 196 196 196 336 336 336 T T F <--- 1 020-039 196 196 196 336 336 336 F F F 2 040-059 197 197 197 338 338 338 F F F 3 060-079 195 195 195 335 335 335 F F F 4 080-099 193 193 193 333 333 333 F F F 5 100-119 198 198 194 342 342 334 F F F <--- 6 120-139 192 192 192 330 330 330 F F F 7 140-159 191 191 191 329 329 329 F F F 8 160-179 197 197 197 338 338 338 F F F 9 180-199 198 198 197 340 340 338 F F F <--- 10 200-219 198 198 197 340 340 338 F F F <--- 11 220-239 198 198 193 340 340 333 F F F <--- 12 240-259 193 193 193 333 333 333 F F F 13 260-279 192 192 192 330 330 330 F F F 14 280-299 196 196 196 337 337 337 F F F 15 300-319 196 196 196 337 337 337 F F F 16 320-339 195 195 195 335 335 335 F F F 17 340-359 196 196 196 336 336 336 F F F 18 360-379 191 191 191 329 329 329 F F F 19 380-399 198 198 195 342 342 335 F F F <--- 20 400-419 195 195 195 335 335 335 F F F 21 420-439 196 196 196 336 336 336 F F F 22 440-459 198 198 196 340 340 336 F F F <--- 23 460-479 196 196 196 337 337 337 F F F 24 480-499 198 198 197 342 342 338 F F F <--- 25 500-519 195 195 195 335 335 335 F F F 26 520-539 191 191 191 329 329 329 F F F 27 540-559 197 197 197 338 338 338 F F F 28 560-579 198 198 195 340 340 335 T T T <--- 29 580-599 198 198 184 340 340 313 F F F <--- Max gen: 198, Max flush: 342 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 99 Verifying data now No change after verify Loop: 99 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 199 199 199 343 343 343 F F F 1 020-039 196 196 196 336 336 336 F F F 2 040-059 197 197 197 338 338 338 F F F 3 060-079 195 195 195 335 335 335 F F F 4 080-099 193 193 193 333 333 333 F F F 5 100-119 198 198 198 342 342 342 F F F 6 120-139 192 192 192 330 330 330 F F F 7 140-159 191 191 191 329 329 329 F F F 8 160-179 197 197 197 338 338 338 F F F 9 180-199 198 198 198 340 340 340 F F F 10 200-219 198 198 198 340 340 340 F F F 11 220-239 198 198 198 340 340 340 F F F 12 240-259 193 193 193 333 333 333 F F F 13 260-279 192 192 192 330 330 330 F F F 14 280-299 196 196 196 337 337 337 F F F 15 300-319 196 196 196 337 337 337 F F F 16 320-339 195 195 195 335 335 335 F F F 17 340-359 196 196 196 336 336 336 F F F 18 360-379 191 191 191 329 329 329 F F F 19 380-399 198 198 198 342 342 342 F F F 20 400-419 195 195 195 335 335 335 F F F 21 420-439 196 196 196 336 336 336 F F F 22 440-459 198 198 198 340 340 340 F F F 23 460-479 196 196 196 337 337 337 F F F 24 480-499 198 198 198 342 342 342 F F F 25 500-519 195 195 195 335 335 335 F F F 26 520-539 191 191 191 329 329 329 F F F 27 540-559 197 197 197 338 338 338 F F F 28 560-579 199 199 199 343 343 343 F F F 29 580-599 198 198 198 340 340 340 F F F Max gen: 199, Max flush: 343 7:28 Test duration Test completed real 7:27.289533879 user 25.251070078 sys 13.934838932 trap 0.018836119 tflt 0.060804247 dflt 0.069445059 kflt 0.003694892 lock 11:52:07.795646602 slp 36:10.058529283 lat 2.265821619 stop 0.019159181